euredact 0.6.0 → 0.7.1

package/dist/cjs/rules/validators.js

@@ -32,7 +32,10 @@ exports.validateSpanishDni = validateSpanishDni;
 exports.validateSpanishNie = validateSpanishNie;
 exports.validatePortugueseNif = validatePortugueseNif;
 exports.validatePolishPesel = validatePolishPesel;
+exports.validatePolishNrb = validatePolishNrb;
 exports.validatePolishNip = validatePolishNip;
+exports.validatePolishIdCard = validatePolishIdCard;
+exports.validatePolishRegon = validatePolishRegon;
 exports.validateCzechBirthNumber = validateCzechBirthNumber;
 exports.validateRomanianCnp = validateRomanianCnp;
 exports.validateHungarianTaj = validateHungarianTaj;
@@ -538,6 +541,13 @@ function validatePolishPesel(candidate) {
         total += d[i] * weights[i];
     return d[10] === (10 - (total % 10)) % 10;
 }
+/** Polish domestic account number (NRB): the PL IBAN without its "PL", so the IBAN's mod-97 decides it (rules-engine#93). */
+function validatePolishNrb(candidate) {
+    const c = candidate.replace(/[\s\-]/g, "");
+    if (!/^\d{26}$/.test(c))
+        return false;
+    return validateIban("PL" + c);
+}
 function validatePolishNip(candidate) {
     const c = candidate.replace(/[\s\-]/g, "");
     if (c.length !== 10 || !/^\d{10}$/.test(c))
@@ -549,6 +559,43 @@ function validatePolishNip(candidate) {
         total += d[i] * weights[i];
     return total % 11 === d[9];
 }
+/**
+ * Polish identity card (dowód osobisty): 3 letters + 6 digits, the first digit
+ * a check digit. Letters A=10 … Z=35, weights 7,3,1,9,7,3,1,7,3: the weighted
+ * sum of all nine is 0 mod 10 exactly when the check digit is right (rules-engine#75).
+ */
+function validatePolishIdCard(candidate) {
+    const c = candidate.replace(/\s/g, "").toUpperCase();
+    if (!/^[A-Z]{3}\d{6}$/.test(c))
+        return false;
+    const weights = [7, 3, 1, 9, 7, 3, 1, 7, 3];
+    let total = 0;
+    for (let i = 0; i < 9; i++) {
+        const ch = c[i];
+        total += (/[A-Z]/.test(ch) ? ch.charCodeAt(0) - 55 : Number(ch)) * weights[i];
+    }
+    return total % 10 === 0;
+}
+/**
+ * Polish REGON: 9 digits, or 14 for a local unit, each with a mod-11 check (a
+ * remainder of 10 counts as 0). A 14-digit REGON begins with its parent's valid
+ * 9-digit REGON (rules-engine#76).
+ */
+function validatePolishRegon(candidate) {
+    const c = candidate.replace(/[\s\-]/g, "");
+    if (!/^\d+$/.test(c) || (c.length !== 9 && c.length !== 14))
+        return false;
+    const check = (digits, weights) => {
+        let total = 0;
+        for (let i = 0; i < weights.length; i++)
+            total += Number(digits[i]) * weights[i];
+        const rest = total % 11;
+        return (rest === 10 ? 0 : rest) === Number(digits[weights.length]);
+    };
+    if (!check(c.slice(0, 9), [8, 9, 2, 3, 4, 5, 6, 7]))
+        return false;
+    return c.length === 9 || check(c, [2, 4, 8, 5, 0, 9, 7, 3, 6, 1, 2, 4, 8]);
+}
 // Days per month for the rodné číslo date check. February is 29 because the
 // century is not recoverable from a two-digit year, so a leap year cannot be
 // ruled out -- being permissive by one day is the right direction here.
@@ -799,6 +846,9 @@ exports.VALIDATORS = {
     portuguese_nif: validatePortugueseNif,
     polish_pesel: validatePolishPesel,
     polish_nip: validatePolishNip,
+    polish_nrb: validatePolishNrb,
+    polish_id_card: validatePolishIdCard,
+    polish_regon: validatePolishRegon,
     czech_birth_number: validateCzechBirthNumber,
     romanian_cnp: validateRomanianCnp,
     hungarian_taj: validateHungarianTaj,

package/dist/cjs/sdk.js

@@ -35,8 +35,11 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EuRedact = exports.TokenMapper = exports.TOKEN_SUFFIX_LENGTH = exports.TOKEN_ALPHABET = void 0;
 exports.applyReplacements = applyReplacements;
+exports.maskForCloud = maskForCloud;
+exports.ontoOriginal = ontoOriginal;
 exports.restore = restore;
-const node_crypto_1 = require("node:crypto");
+exports.normalizeExcludeTypes = normalizeExcludeTypes;
+const platform_js_1 = require("./platform.js");
 const cache_js_1 = require("./cache.js");
 const normalizer_js_1 = require("./normalizer.js");
 const engine_js_1 = require("./rules/engine.js");
@@ -98,6 +101,7 @@ function applyReplacements(text, detections, labelFor) {
  * do not list it". A token or a numbered label would be read as ordinary text,
  * so the format on the wire is a model contract and not an output option.
  */
+/** @internal Shared with cloud/batches.ts. */
 function maskForCloud(text, detections) {
     const parts = [];
     const labels = [];
@@ -148,6 +152,7 @@ function labelBefore(labels, offset, strict) {
  * a span cannot be placed, and masking where it points would cover the wrong
  * characters and leave the right ones in the clear.
  */
+/** @internal Shared with cloud/batches.ts. */
 function ontoOriginal(spans, masked, text, labels, Unplaceable) {
     const placed = [];
     for (const span of spans) {
@@ -177,11 +182,12 @@ function ontoOriginal(spans, masked, text, labels, Unplaceable) {
     return placed;
 }
 /**
- * Characters a token suffix is drawn from. No vowels, so a suffix never spells
- * a word; no 0/1/I/O, so it survives being read back by a person; no
- * underscore, so the type prefix stays unambiguous.
+ * Characters a token suffix is drawn from. No vowels (and no Y), so a suffix
+ * never spells a word; no 0/1/I/O, so it survives being read back by a person;
+ * no underscore, so the type prefix stays unambiguous. A, E, U and Y were in it
+ * until rules-engine#55, against this comment.
  */
-exports.TOKEN_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
+exports.TOKEN_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ23456789";
 exports.TOKEN_SUFFIX_LENGTH = 4;
 const TOKEN_MAX_DRAWS = 100;
 function escapeRegExp(s) {
@@ -229,7 +235,7 @@ class TokenMapper {
                 }
                 let suffix = "";
                 for (let i = 0; i < exports.TOKEN_SUFFIX_LENGTH; i++) {
-                    suffix += exports.TOKEN_ALPHABET[(0, node_crypto_1.randomInt)(exports.TOKEN_ALPHABET.length)];
+                    suffix += exports.TOKEN_ALPHABET[(0, platform_js_1.randomIndex)(exports.TOKEN_ALPHABET.length)];
                 }
                 token = prefix + suffix;
             } while (this.taken.has(token));
@@ -389,6 +395,70 @@ function applyAllowlist(text, detections, allowed, domains) {
     }
     return [kept, exempted];
 }
+const KNOWN_TYPES = new Set(Object.values(types_js_1.EntityType));
+/**
+ * The entity types a caller asked to leave in the text, by canonical name.
+ *
+ * Names are matched case-insensitively and legacy names resolve (`IBAN` is
+ * `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A custom pattern's name counts
+ * once it is registered. Anything else throws: a typo in a type meant to be
+ * switched off would otherwise switch nothing off (rules-engine#125).
+ */
+function normalizeExcludeTypes(values, customNames = new Set(), param = "excludeTypes") {
+    const out = new Set();
+    if (values === null || values === undefined)
+        return out;
+    checkAllowlistArg(values, param);
+    for (const v of values) {
+        if (typeof v !== "string") {
+            throw new TypeError(`${param} entries must be type names, not ${typeof v}`);
+        }
+        if (customNames.has(v)) {
+            out.add(v);
+            continue;
+        }
+        const name = (0, types_js_1.canonicalType)(v);
+        if (!KNOWN_TYPES.has(name)) {
+            throw new Error(`unknown entity type ${JSON.stringify(v)} in ${param}. Known types: ` +
+                [...KNOWN_TYPES].sort().join(", ") +
+                (customNames.size ? `; custom patterns: ${[...customNames].sort().join(", ")}` : ""));
+        }
+        out.add(name);
+    }
+    return out;
+}
+/**
+ * Split off the detections of the types the caller excluded. Applied to each
+ * span's final type, after ranking: dropping an excluded type's candidates
+ * earlier would let the next-best type claim the same span.
+ */
+function applyTypeExclusion(text, detections, excluded) {
+    if (excluded.size === 0)
+        return [detections, []];
+    const kept = [];
+    const exempted = [];
+    for (const d of detections) {
+        const name = String(d.entityType);
+        if (excluded.has(name)) {
+            exempted.push({
+                entityType: d.entityType, start: d.start, end: d.end,
+                text: text.slice(d.start, d.end), rule: name, ruleKind: "type",
+            });
+        }
+        else
+            kept.push(d);
+    }
+    return [kept, exempted];
+}
+/** Allowlist first, then excluded types; every exemption in text order. */
+function exempt(text, detections, allowed, domains, excluded) {
+    const [afterValues, byValue] = applyAllowlist(text, detections, allowed, domains);
+    const [kept, byType] = applyTypeExclusion(text, afterValues, excluded);
+    const exempted = [...byValue, ...byType];
+    if (byValue.length && byType.length)
+        exempted.sort((a, b) => a.start - b.start || b.end - a.end);
+    return [kept, exempted];
+}
 /** Two label schemes for the same spans cannot both apply. */
 function checkLabelOptions(options) {
     if (options.tokenize && options.referentialIntegrity) {
@@ -469,19 +539,26 @@ class EuRedact {
     maxInputLength;
     allowlist;
     allowlistDomains;
+    customNames = new Set();
+    excludeTypes;
     /**
      * @param options.maxInputLength Longest document `redact` accepts, in characters.
      * @param options.allowlist Values never to redact, for every call on this
      *   instance — an organisation's own name, its own addresses. Merged with
      *   the per-call `allowlist`. Matched whole, case-insensitively.
+     * @param options.excludeTypes Entity types never reported or masked, for
+     *   every call on this instance, e.g. `["JOB_TITLE", "FINANCIAL_AMOUNT"]`.
+     *   Merged with the per-call value. Unknown names throw.
      */
     constructor(options) {
         this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
         this.allowlist = normalizeAllowlist(options?.allowlist);
         this.allowlistDomains = normalizeDomains(options?.allowlistDomains);
+        this.excludeTypes = normalizeExcludeTypes(options?.excludeTypes);
     }
     addCustomPattern(name, pattern) {
         this.engine.addCustomPattern(name, pattern);
+        this.customNames.add(name);
         this.cache.clear();
     }
     /** Clear the result cache and referential integrity mappings.
@@ -497,6 +574,13 @@ class EuRedact {
             allowed.set(k, v);
         return allowed;
     }
+    /** The instance's excluded types merged with a call's. */
+    excludedFor(options) {
+        const excluded = new Set(this.excludeTypes);
+        for (const t of normalizeExcludeTypes(options.excludeTypes, this.customNames))
+            excluded.add(t);
+        return excluded;
+    }
     /** The instance domain list merged with a call's. */
     domainsFor(options) {
         const domains = new Map(this.allowlistDomains);
@@ -553,22 +637,26 @@ class EuRedact {
         }
         const allowed = this.allowedFor(options);
         const domains = this.domainsFor(options);
+        const excluded = this.excludedFor(options);
         const client = new CloudClient();
         // Dates are always on, whatever detectDates says: that is the rules output
         // the model was trained against, and a date of birth the rules can place
         // has no reason to travel. It is the one ignored option that is safe to
         // ignore — it can only cause MORE to be detected, never less.
         //
-        // No allowlist and no tokens on this pass. An exemption says what the
-        // caller wants back, not what may leave; both are applied below, after the
+        // No allowlist, no excluded types and no tokens on this pass. An exemption
+        // says what the caller wants back, not what may leave: an excluded EMAIL is
+        // still masked in the request. All three are applied below, after the
         // response, to the merged spans.
         const local = this.redactRules(text, { countries, detectDates: true, cache: options.cache }, new Map(), new Map());
         const [masked, labels] = maskForCloud(text, local.detections);
         // The client converts the service's code-point offsets to UTF-16 units of
         // the text it sent, so everything from here on is in one unit.
-        const remote = await client.redact(masked, { country: countries[0] });
+        const remote = await client.redact(masked, {
+            country: countries[0], idempotencyKey: options.idempotencyKey,
+        });
         // A new array throughout: `local` may be the cached rules result.
-        const [detections, exempted] = applyAllowlist(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains);
+        const [detections, exempted] = exempt(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains, excluded);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
         const tokenMapper = options.tokenize ? new TokenMapper(text, detections) : null;
         return {
@@ -578,6 +666,7 @@ class EuRedact {
             source: "cloud",
             tokens: tokenMapper ? tokenMapper.tokens : {},
             exempted,
+            ...(remote.cloud ? { cloud: remote.cloud } : {}),
         };
     }
     redact(text, options = {}) {
@@ -586,6 +675,7 @@ class EuRedact {
         checkLabelOptions(options);
         const allowed = this.allowedFor(options);
         const domains = this.domainsFor(options);
+        const excluded = this.excludedFor(options);
         const requestedMode = options.mode ?? "rules";
         if (requestedMode === "cloud") {
             // Cannot be served synchronously: the cloud tier is a network call, and
@@ -599,10 +689,13 @@ class EuRedact {
         if (requestedMode !== "rules") {
             throw new Error(`unknown mode ${JSON.stringify(requestedMode)}: expected "rules" or "cloud"`);
         }
-        return this.redactRules(text, options, allowed, domains);
+        if (options.idempotencyKey !== undefined) {
+            throw new Error('idempotencyKey applies to mode: "cloud" only: a rules-only redaction makes no request to deduplicate');
+        }
+        return this.redactRules(text, options, allowed, domains, excluded);
     }
     /** The local pipeline: normalise, detect, exempt, mask. */
-    redactRules(text, options, allowed, domains) {
+    redactRules(text, options, allowed, domains, excluded = new Set()) {
         if (text.length > this.maxInputLength) {
             throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
                 `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
@@ -625,7 +718,8 @@ class EuRedact {
         // JSON rather than a joined string: an entry may itself contain the
         // separator, and two different lists must never share a key.
         const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
-            + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "");
+            + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "")
+            + (excluded.size ? "|excl=" + JSON.stringify([...excluded].sort()) : "");
         const cacheMode = `rules|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
         let cacheKey;
         if (cache) {
@@ -649,7 +743,7 @@ class EuRedact {
             detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
         }
         let exempted;
-        [detections, exempted] = applyAllowlist(text, detections, allowed, domains);
+        [detections, exempted] = exempt(text, detections, allowed, domains, excluded);
         detections.sort((a, b) => a.start - b.start || b.end - a.end);
         const tokenMapper = tokenize ? new TokenMapper(text, detections) : null;
         const redacted = applyReplacements(text, detections, this.labelFor(referentialIntegrity, tokenMapper));

package/dist/cjs/types.js

@@ -79,7 +79,8 @@ var EntityType;
     EntityType["BIOMETRIC_REF"] = "BIOMETRIC_REF";
     EntityType["FINANCIAL_AMOUNT"] = "FINANCIAL_AMOUNT";
     EntityType["QUASI_IDENTIFIER"] = "QUASI_IDENTIFIER";
-    EntityType["CREDENTIAL"] = "CREDENTIAL";
+    /** Legacy alias of SECRET: every credential, the model's included, is SECRET (rules-engine#111). */
+    EntityType["CREDENTIAL"] = "SECRET";
     EntityType["URL"] = "URL";
     EntityType["OTHER"] = "OTHER";
 })(EntityType || (exports.EntityType = EntityType = {}));
@@ -93,6 +94,8 @@ exports.LEGACY_TYPE_ALIASES = {
     NAME: "PERSON_NAME",
     STREET_ADDRESS: "ADDRESS",
     NATIONALITY_ETHNICITY: "SENSITIVE_ATTRIBUTE",
+    // The model labels prose credentials CREDENTIAL; the SDK reports every credential as SECRET (rules-engine#111).
+    CREDENTIAL: "SECRET",
 };
 /**
  * Resolve a possibly-legacy type name to its canonical form.

package/dist/esm/cache.js

@@ -1,4 +1,4 @@
-import { createHash } from "node:crypto";
+import { sha256Hex } from "./platform.js";
 // Default ceiling on retained characters, not entries. A count-only cap of
 // 1024 entries says nothing about memory: 1024 cached 1 MB documents is a
 // gigabyte of PII-bearing text held live. ~16M characters is a few tens of MB.
@@ -33,13 +33,7 @@ export class ResultCache {
         // deployment an attacker who can guess a victim's document could submit a
         // same-length colliding one and be served the victim's cached result,
         // detections and raw matched PII included.
-        return createHash("sha256")
-            .update(text)
-            .update("|")
-            .update(sorted.join("|"))
-            .update("|")
-            .update(mode)
-            .digest("hex");
+        return sha256Hex([text, "|", sorted.join("|"), "|", mode]);
     }
     get(key) {
         if (!this.enabled)

package/dist/esm/cloud/account.d.ts

@@ -0,0 +1,155 @@
+/**
+ * [CLOUD EXTENSION] The account and job endpoints an API key may call.
+ *
+ * Everything a customer's key can do on the service is reachable from the SDK
+ * (rules-engine#89): redaction (`CloudClient`, `redactAsync`), batches
+ * (`Batches`), a past job (`Jobs`) and the account itself (`Account`). What
+ * needs a signed-in person -- logging in, minting keys, changing settings,
+ * members, accepting terms -- is not here, because a key cannot do it.
+ *
+ * Every typed value carries `raw`, the service's JSON, so a field the SDK does
+ * not know yet is still reachable.
+ */
+import { type CloudConfig } from "./config.js";
+import type { RedactResult } from "../types.js";
+type Json = Record<string, unknown>;
+/** `GET /v1/account`: plan limits, retention and usage against quota. */
+export interface AccountSummary {
+    tenantId: string | null;
+    tenantName: string | null;
+    active: boolean | null;
+    memberSince: string | null;
+    role: string | null;
+    /** `"ttl"`, or `"none"` when text is discarded on delivery. */
+    retentionMode: string | null;
+    payloadTtlHours: number | null;
+    documentsToday: number | null;
+    documentsThisMonth: number | null;
+    tokensThisMonth: number | null;
+    failures24h: number | null;
+    /** `null` for an account without a daily quota. */
+    dailyQuota: number | null;
+    quotaRemaining: number | null;
+    activeKeys: number | null;
+    batchesAvailable: boolean | null;
+    raw: Json;
+}
+/** `GET /v1/account/credits`: the balance, in credits. */
+export interface Credits {
+    balance: number | null;
+    granted: number | null;
+    spent: number | null;
+    unit: string | null;
+    raw: Json;
+}
+/** One line of the credit history: movements rolled up per minute, reason and direction, newest first. */
+export interface CreditEntry {
+    at: string | null;
+    /** Negative for a debit. */
+    delta: number | null;
+    reason: string | null;
+    count: number | null;
+    jobId: string | null;
+    sourceRef: string | null;
+    raw: Json;
+}
+/** One day of `GET /v1/account/usage`, or of one key's series. */
+export interface UsageDay {
+    day: string | null;
+    documents: number | null;
+    tokens: number | null;
+    failures: number | null;
+    rulesOnly: number | null;
+    raw: Json;
+}
+/** One key of `GET /v1/account/usage/by-key`; the rest are rolled into one with `keyId` null. */
+export interface KeyUsage {
+    keyId: number | null;
+    name: string | null;
+    documents: number | null;
+    tokens: number | null;
+    failures: number | null;
+    series: UsageDay[];
+    raw: Json;
+}
+/** One key of `GET /v1/account/keys`. Never its secret. */
+export interface ApiKey {
+    id: number | null;
+    name: string | null;
+    createdBy: string | null;
+    createdAt: string | null;
+    lastUsedAt: string | null;
+    revokedAt: string | null;
+    active: boolean | null;
+    raw: Json;
+}
+/** The answer to `POST /v1/account/keys/{id}/revoke`. */
+export interface KeyRevocation {
+    id: number | null;
+    keysRemaining: number | null;
+    /** True when no active key is left: nothing can call the service until a person mints one. */
+    lockedOut: boolean | null;
+    /** True when the key revoked is the one that made this call. */
+    selfRevoked: boolean | null;
+    raw: Json;
+}
+/** `GET /v1/jobs/{id}`: a job's state, and its result once it has one. */
+export interface Job {
+    jobId: string;
+    /** `queued`, `running` or `succeeded`. A failed or rejected job throws instead. */
+    status: string;
+    createdAt: string | null;
+    /**
+     * Set when the job succeeded. Its text and offsets are those of the text the
+     * service received -- under the local-first cloud mode, the locally masked
+     * text, not the caller's original.
+     */
+    result: RedactResult | null;
+    raw: Json;
+}
+export interface ApiClientOptions {
+    /** Injectable for tests. Defaults to the platform `fetch`. */
+    fetchImpl?: typeof fetch;
+}
+/** A JSON client: GETs are retried like the redact call; writes are sent once. */
+declare class ApiClient {
+    readonly config: CloudConfig;
+    private readonly fetchImpl?;
+    constructor(config?: CloudConfig | null, options?: ApiClientOptions);
+    protected call(method: "GET" | "POST", path: string, params?: Record<string, number>, retry?: boolean): Promise<Json>;
+}
+/**
+ * The account an API key belongs to: summary, credits, usage and keys.
+ *
+ * These calls pass with an API key while the service accepts account calls
+ * without a browser `Origin` (its `EUREDACT_CONSOLE_REQUIRE_ORIGIN=0`); a
+ * refusal arrives as a `CloudError` with status 403.
+ */
+export declare class Account extends ApiClient {
+    summary(): Promise<AccountSummary>;
+    credits(): Promise<Credits>;
+    /** Recent credit movements, newest first (1-500). */
+    creditHistory(limit?: number): Promise<CreditEntry[]>;
+    /** Documents, tokens and failures per day over the last `days`. */
+    usage(days?: number): Promise<UsageDay[]>;
+    /** The same window, split by the key that submitted the work. */
+    usageByKey(days?: number): Promise<KeyUsage[]>;
+    keys(): Promise<ApiKey[]>;
+    /**
+     * Revoke a key, e.g. one that leaked: the one write a key may make. Sent once,
+     * never retried. Already revoked throws a `CloudError` with status 409; a key
+     * that does not exist, or is another account's, throws `NotFoundError`.
+     * Revoking the key this client uses works, and is reported by `selfRevoked`.
+     */
+    revokeKey(keyId: number): Promise<KeyRevocation>;
+}
+/** Past jobs, by id. */
+export declare class Jobs extends ApiClient {
+    /**
+     * A job's state, and its result if it has one. Throws `ResultExpiredError`
+     * when the job succeeded but its result is no longer retained, and
+     * `NotFoundError` for an unknown job (or another account's).
+     */
+    retrieve(jobId: string): Promise<Job>;
+}
+export {};