euredact 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/NOTICE +11 -0
  2. package/README.md +455 -44
  3. package/dist/cjs/cache.js +2 -8
  4. package/dist/cjs/cloud/account.js +180 -0
  5. package/dist/cjs/cloud/batches.js +469 -0
  6. package/dist/cjs/cloud/client.js +124 -14
  7. package/dist/cjs/cloud/config.js +28 -2
  8. package/dist/cjs/cloud/errors.js +39 -2
  9. package/dist/cjs/cloud/index.js +16 -1
  10. package/dist/cjs/index.js +9 -1
  11. package/dist/cjs/platform.browser.js +98 -0
  12. package/dist/cjs/platform.js +140 -0
  13. package/dist/cjs/rules/bicGleif.js +57 -0
  14. package/dist/cjs/rules/bicRegistry.js +37 -11
  15. package/dist/cjs/rules/countries/index.js +83 -13
  16. package/dist/cjs/rules/cues.js +29 -9
  17. package/dist/cjs/rules/engine.js +14 -6
  18. package/dist/cjs/rules/suppressors.js +479 -26
  19. package/dist/cjs/rules/validators.js +50 -0
  20. package/dist/cjs/sdk.js +108 -14
  21. package/dist/cjs/types.js +4 -1
  22. package/dist/esm/cache.js +2 -8
  23. package/dist/esm/cloud/account.d.ts +155 -0
  24. package/dist/esm/cloud/account.js +175 -0
  25. package/dist/esm/cloud/batches.d.ts +166 -0
  26. package/dist/esm/cloud/batches.js +464 -0
  27. package/dist/esm/cloud/client.d.ts +83 -1
  28. package/dist/esm/cloud/client.js +119 -22
  29. package/dist/esm/cloud/config.d.ts +6 -0
  30. package/dist/esm/cloud/config.js +27 -2
  31. package/dist/esm/cloud/errors.d.ts +26 -1
  32. package/dist/esm/cloud/errors.js +35 -1
  33. package/dist/esm/cloud/index.d.ts +4 -2
  34. package/dist/esm/cloud/index.js +4 -2
  35. package/dist/esm/index.d.ts +2 -2
  36. package/dist/esm/index.js +1 -1
  37. package/dist/esm/platform.browser.d.ts +27 -0
  38. package/dist/esm/platform.browser.js +92 -0
  39. package/dist/esm/platform.d.ts +20 -0
  40. package/dist/esm/platform.js +101 -0
  41. package/dist/esm/rules/bicGleif.d.ts +4 -0
  42. package/dist/esm/rules/bicGleif.js +53 -0
  43. package/dist/esm/rules/bicRegistry.d.ts +25 -11
  44. package/dist/esm/rules/bicRegistry.js +36 -11
  45. package/dist/esm/rules/countries/index.js +83 -13
  46. package/dist/esm/rules/cues.d.ts +1 -1
  47. package/dist/esm/rules/cues.js +29 -9
  48. package/dist/esm/rules/engine.js +14 -6
  49. package/dist/esm/rules/suppressors.d.ts +1 -0
  50. package/dist/esm/rules/suppressors.js +478 -25
  51. package/dist/esm/rules/validators.d.ts +14 -0
  52. package/dist/esm/rules/validators.js +47 -0
  53. package/dist/esm/sdk.d.ts +77 -4
  54. package/dist/esm/sdk.js +108 -17
  55. package/dist/esm/types.d.ts +64 -6
  56. package/dist/esm/types.js +4 -1
  57. package/package.json +13 -4
@@ -65,7 +65,21 @@ export declare function validateSpanishDni(candidate: string): boolean;
65
65
  export declare function validateSpanishNie(candidate: string): boolean;
66
66
  export declare function validatePortugueseNif(candidate: string): boolean;
67
67
  export declare function validatePolishPesel(candidate: string): boolean;
68
+ /** Polish domestic account number (NRB): the PL IBAN without its "PL", so the IBAN's mod-97 decides it (rules-engine#93). */
69
+ export declare function validatePolishNrb(candidate: string): boolean;
68
70
  export declare function validatePolishNip(candidate: string): boolean;
71
+ /**
72
+ * Polish identity card (dowód osobisty): 3 letters + 6 digits, the first digit
73
+ * a check digit. Letters A=10 … Z=35, weights 7,3,1,9,7,3,1,7,3: the weighted
74
+ * sum of all nine is 0 mod 10 exactly when the check digit is right (rules-engine#75).
75
+ */
76
+ export declare function validatePolishIdCard(candidate: string): boolean;
77
+ /**
78
+ * Polish REGON: 9 digits, or 14 for a local unit, each with a mod-11 check (a
79
+ * remainder of 10 counts as 0). A 14-digit REGON begins with its parent's valid
80
+ * 9-digit REGON (rules-engine#76).
81
+ */
82
+ export declare function validatePolishRegon(candidate: string): boolean;
69
83
  export declare function validateCzechBirthNumber(candidate: string): boolean;
70
84
  export declare function validateRomanianCnp(candidate: string): boolean;
71
85
  export declare function validateHungarianTaj(candidate: string): boolean;
@@ -492,6 +492,13 @@ export function validatePolishPesel(candidate) {
492
492
  total += d[i] * weights[i];
493
493
  return d[10] === (10 - (total % 10)) % 10;
494
494
  }
495
+ /** Polish domestic account number (NRB): the PL IBAN without its "PL", so the IBAN's mod-97 decides it (rules-engine#93). */
496
+ export function validatePolishNrb(candidate) {
497
+ const c = candidate.replace(/[\s\-]/g, "");
498
+ if (!/^\d{26}$/.test(c))
499
+ return false;
500
+ return validateIban("PL" + c);
501
+ }
495
502
  export function validatePolishNip(candidate) {
496
503
  const c = candidate.replace(/[\s\-]/g, "");
497
504
  if (c.length !== 10 || !/^\d{10}$/.test(c))
@@ -503,6 +510,43 @@ export function validatePolishNip(candidate) {
503
510
  total += d[i] * weights[i];
504
511
  return total % 11 === d[9];
505
512
  }
513
+ /**
514
+ * Polish identity card (dowód osobisty): 3 letters + 6 digits, the first digit
515
+ * a check digit. Letters A=10 … Z=35, weights 7,3,1,9,7,3,1,7,3: the weighted
516
+ * sum of all nine is 0 mod 10 exactly when the check digit is right (rules-engine#75).
517
+ */
518
+ export function validatePolishIdCard(candidate) {
519
+ const c = candidate.replace(/\s/g, "").toUpperCase();
520
+ if (!/^[A-Z]{3}\d{6}$/.test(c))
521
+ return false;
522
+ const weights = [7, 3, 1, 9, 7, 3, 1, 7, 3];
523
+ let total = 0;
524
+ for (let i = 0; i < 9; i++) {
525
+ const ch = c[i];
526
+ total += (/[A-Z]/.test(ch) ? ch.charCodeAt(0) - 55 : Number(ch)) * weights[i];
527
+ }
528
+ return total % 10 === 0;
529
+ }
530
+ /**
531
+ * Polish REGON: 9 digits, or 14 for a local unit, each with a mod-11 check (a
532
+ * remainder of 10 counts as 0). A 14-digit REGON begins with its parent's valid
533
+ * 9-digit REGON (rules-engine#76).
534
+ */
535
+ export function validatePolishRegon(candidate) {
536
+ const c = candidate.replace(/[\s\-]/g, "");
537
+ if (!/^\d+$/.test(c) || (c.length !== 9 && c.length !== 14))
538
+ return false;
539
+ const check = (digits, weights) => {
540
+ let total = 0;
541
+ for (let i = 0; i < weights.length; i++)
542
+ total += Number(digits[i]) * weights[i];
543
+ const rest = total % 11;
544
+ return (rest === 10 ? 0 : rest) === Number(digits[weights.length]);
545
+ };
546
+ if (!check(c.slice(0, 9), [8, 9, 2, 3, 4, 5, 6, 7]))
547
+ return false;
548
+ return c.length === 9 || check(c, [2, 4, 8, 5, 0, 9, 7, 3, 6, 1, 2, 4, 8]);
549
+ }
506
550
  // Days per month for the rodné číslo date check. February is 29 because the
507
551
  // century is not recoverable from a two-digit year, so a leap year cannot be
508
552
  // ruled out -- being permissive by one day is the right direction here.
@@ -753,6 +797,9 @@ export const VALIDATORS = {
753
797
  portuguese_nif: validatePortugueseNif,
754
798
  polish_pesel: validatePolishPesel,
755
799
  polish_nip: validatePolishNip,
800
+ polish_nrb: validatePolishNrb,
801
+ polish_id_card: validatePolishIdCard,
802
+ polish_regon: validatePolishRegon,
756
803
  czech_birth_number: validateCzechBirthNumber,
757
804
  romanian_cnp: validateRomanianCnp,
758
805
  hungarian_taj: validateHungarianTaj,
package/dist/esm/sdk.d.ts CHANGED
@@ -22,11 +22,51 @@ type LabelFor = (det: Detection, slice: string) => string;
22
22
  */
23
23
  export declare function applyReplacements(text: string, detections: Detection[], labelFor: LabelFor): string;
24
24
  /**
25
- * Characters a token suffix is drawn from. No vowels, so a suffix never spells
26
- * a word; no 0/1/I/O, so it survives being read back by a person; no
27
- * underscore, so the type prefix stays unambiguous.
25
+ * One label in the text sent to the cloud tier: where it sits in the masked
26
+ * text, and the extent of the original it replaced —
27
+ * `[maskedStart, maskedEnd, start, end]`.
28
28
  */
29
- export declare const TOKEN_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
29
+ export type WireLabel = [number, number, number, number];
30
+ /**
31
+ * Bracket-mask `text` and record where each label landed.
32
+ *
33
+ * Returns what is sent to the cloud tier, plus one `WireLabel` per label.
34
+ * Everything between labels is copied unchanged, so the labels are all it
35
+ * takes to carry an offset from the masked text back to the original.
36
+ *
37
+ * Always `[TYPE]`, whatever label scheme the caller asked for. That is the
38
+ * syntax the model was trained on, where a placeholder means "already handled,
39
+ * do not list it". A token or a numbered label would be read as ordinary text,
40
+ * so the format on the wire is a model contract and not an output option.
41
+ */
42
+ /** @internal Shared with cloud/batches.ts. */
43
+ export declare function maskForCloud(text: string, detections: Detection[]): [string, WireLabel[]];
44
+ /**
45
+ * Carry the service's spans from the masked text onto the original.
46
+ *
47
+ * The service only ever saw `masked`, so its offsets index that. An offset in
48
+ * copied text shifts by whatever the labels before it added or removed. An
49
+ * offset inside a label has no counterpart in the original, so it snaps
50
+ * outward to the edge of the value the label replaced: an address the model
51
+ * reports around `[POSTAL_CODE]` covers the postal code, never half of it.
52
+ * Over-masking is the safe direction.
53
+ *
54
+ * A span lying wholly inside one label describes the label, not the document.
55
+ * It is dropped; the local detection behind the label stands.
56
+ *
57
+ * Throws `Unplaceable` when a span does not match the text that was sent. Such
58
+ * a span cannot be placed, and masking where it points would cover the wrong
59
+ * characters and leave the right ones in the clear.
60
+ */
61
+ /** @internal Shared with cloud/batches.ts. */
62
+ export declare function ontoOriginal(spans: Detection[], masked: string, text: string, labels: WireLabel[], Unplaceable: new (message: string) => Error): Detection[];
63
+ /**
64
+ * Characters a token suffix is drawn from. No vowels (and no Y), so a suffix
65
+ * never spells a word; no 0/1/I/O, so it survives being read back by a person;
66
+ * no underscore, so the type prefix stays unambiguous. A, E, U and Y were in it
67
+ * until rules-engine#55, against this comment.
68
+ */
69
+ export declare const TOKEN_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ23456789";
30
70
  export declare const TOKEN_SUFFIX_LENGTH = 4;
31
71
  /**
32
72
  * Maps each distinct PII value in one call to a reversible token.
@@ -64,6 +104,15 @@ export declare class TokenMapper {
64
104
  * restored, since leaving a token behind is the worse failure.
65
105
  */
66
106
  export declare function restore(text: string, tokens: Record<string, string>): string;
107
+ /**
108
+ * The entity types a caller asked to leave in the text, by canonical name.
109
+ *
110
+ * Names are matched case-insensitively and legacy names resolve (`IBAN` is
111
+ * `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A custom pattern's name counts
112
+ * once it is registered. Anything else throws: a typo in a type meant to be
113
+ * switched off would otherwise switch nothing off (rules-engine#125).
114
+ */
115
+ export declare function normalizeExcludeTypes(values: unknown, customNames?: ReadonlySet<string>, param?: string): Set<string>;
67
116
  export interface RedactOptions {
68
117
  /**
69
118
  * Scope. Detections attributed elsewhere are flagged `outOfScope`, never
@@ -104,8 +153,24 @@ export interface RedactOptions {
104
153
  * EMAIL and URL only, and covers subdomains. Merged with the instance list.
105
154
  */
106
155
  allowlistDomains?: string[] | null;
156
+ /**
157
+ * Entity types to leave in the text: neither in `detections` nor masked, and
158
+ * each such detection reported in `RedactResult.exempted` (`ruleKind:
159
+ * "type"`). Names are case-insensitive and legacy names resolve; unknown
160
+ * names throw. Merged with the instance's list. Default: nothing excluded.
161
+ * In `mode: "cloud"` an excluded rules type is still masked in the text sent
162
+ * to the service — this decides what comes back, not what may leave.
163
+ */
164
+ excludeTypes?: string[] | null;
107
165
  detectDates?: boolean;
108
166
  cache?: boolean;
167
+ /**
168
+ * `mode: "cloud"` only. Sent as the request's `Idempotency-Key`: the same key
169
+ * returns the same job instead of running (and billing) the document again,
170
+ * which is what a caller retrying after its own timeout needs. One is
171
+ * generated per call when omitted.
172
+ */
173
+ idempotencyKey?: string;
109
174
  }
110
175
  export declare class EuRedact {
111
176
  private engine;
@@ -114,16 +179,22 @@ export declare class EuRedact {
114
179
  private maxInputLength;
115
180
  private allowlist;
116
181
  private allowlistDomains;
182
+ private customNames;
183
+ private excludeTypes;
117
184
  /**
118
185
  * @param options.maxInputLength Longest document `redact` accepts, in characters.
119
186
  * @param options.allowlist Values never to redact, for every call on this
120
187
  * instance — an organisation's own name, its own addresses. Merged with
121
188
  * the per-call `allowlist`. Matched whole, case-insensitively.
189
+ * @param options.excludeTypes Entity types never reported or masked, for
190
+ * every call on this instance, e.g. `["JOB_TITLE", "FINANCIAL_AMOUNT"]`.
191
+ * Merged with the per-call value. Unknown names throw.
122
192
  */
123
193
  constructor(options?: {
124
194
  maxInputLength?: number;
125
195
  allowlist?: string[] | null;
126
196
  allowlistDomains?: string[] | null;
197
+ excludeTypes?: string[] | null;
127
198
  });
128
199
  addCustomPattern(name: string, pattern: string): void;
129
200
  /** Clear the result cache and referential integrity mappings.
@@ -131,6 +202,8 @@ export declare class EuRedact {
131
202
  clear(): void;
132
203
  /** The instance allowlist merged with a call's, in comparison form. */
133
204
  private allowedFor;
205
+ /** The instance's excluded types merged with a call's. */
206
+ private excludedFor;
134
207
  /** The instance domain list merged with a call's. */
135
208
  private domainsFor;
136
209
  /** The label function for one call, given its output options. */
package/dist/esm/sdk.js CHANGED
@@ -1,9 +1,9 @@
1
- import { randomInt } from "node:crypto";
1
+ import { randomIndex } from "./platform.js";
2
2
  import { ResultCache } from "./cache.js";
3
3
  import { normalize, mapOffsetToOriginal } from "./normalizer.js";
4
4
  import { RuleEngine } from "./rules/engine.js";
5
5
  import { weightsToRanking } from "./rules/evidence.js";
6
- import { EntityType } from "./types.js";
6
+ import { EntityType, canonicalType } from "./types.js";
7
7
  const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
8
8
  /**
9
9
  * Splice a label over every detection and return the masked text.
@@ -60,7 +60,8 @@ export function applyReplacements(text, detections, labelFor) {
60
60
  * do not list it". A token or a numbered label would be read as ordinary text,
61
61
  * so the format on the wire is a model contract and not an output option.
62
62
  */
63
- function maskForCloud(text, detections) {
63
+ /** @internal Shared with cloud/batches.ts. */
64
+ export function maskForCloud(text, detections) {
64
65
  const parts = [];
65
66
  const labels = [];
66
67
  let pos = 0;
@@ -110,7 +111,8 @@ function labelBefore(labels, offset, strict) {
110
111
  * a span cannot be placed, and masking where it points would cover the wrong
111
112
  * characters and leave the right ones in the clear.
112
113
  */
113
- function ontoOriginal(spans, masked, text, labels, Unplaceable) {
114
+ /** @internal Shared with cloud/batches.ts. */
115
+ export function ontoOriginal(spans, masked, text, labels, Unplaceable) {
114
116
  const placed = [];
115
117
  for (const span of spans) {
116
118
  if (!(0 <= span.start && span.start <= span.end && span.end <= masked.length) ||
@@ -139,11 +141,12 @@ function ontoOriginal(spans, masked, text, labels, Unplaceable) {
139
141
  return placed;
140
142
  }
141
143
  /**
142
- * Characters a token suffix is drawn from. No vowels, so a suffix never spells
143
- * a word; no 0/1/I/O, so it survives being read back by a person; no
144
- * underscore, so the type prefix stays unambiguous.
144
+ * Characters a token suffix is drawn from. No vowels (and no Y), so a suffix
145
+ * never spells a word; no 0/1/I/O, so it survives being read back by a person;
146
+ * no underscore, so the type prefix stays unambiguous. A, E, U and Y were in it
147
+ * until rules-engine#55, against this comment.
145
148
  */
146
- export const TOKEN_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
149
+ export const TOKEN_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ23456789";
147
150
  export const TOKEN_SUFFIX_LENGTH = 4;
148
151
  const TOKEN_MAX_DRAWS = 100;
149
152
  function escapeRegExp(s) {
@@ -191,7 +194,7 @@ export class TokenMapper {
191
194
  }
192
195
  let suffix = "";
193
196
  for (let i = 0; i < TOKEN_SUFFIX_LENGTH; i++) {
194
- suffix += TOKEN_ALPHABET[randomInt(TOKEN_ALPHABET.length)];
197
+ suffix += TOKEN_ALPHABET[randomIndex(TOKEN_ALPHABET.length)];
195
198
  }
196
199
  token = prefix + suffix;
197
200
  } while (this.taken.has(token));
@@ -350,6 +353,70 @@ function applyAllowlist(text, detections, allowed, domains) {
350
353
  }
351
354
  return [kept, exempted];
352
355
  }
356
+ const KNOWN_TYPES = new Set(Object.values(EntityType));
357
+ /**
358
+ * The entity types a caller asked to leave in the text, by canonical name.
359
+ *
360
+ * Names are matched case-insensitively and legacy names resolve (`IBAN` is
361
+ * `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A custom pattern's name counts
362
+ * once it is registered. Anything else throws: a typo in a type meant to be
363
+ * switched off would otherwise switch nothing off (rules-engine#125).
364
+ */
365
+ export function normalizeExcludeTypes(values, customNames = new Set(), param = "excludeTypes") {
366
+ const out = new Set();
367
+ if (values === null || values === undefined)
368
+ return out;
369
+ checkAllowlistArg(values, param);
370
+ for (const v of values) {
371
+ if (typeof v !== "string") {
372
+ throw new TypeError(`${param} entries must be type names, not ${typeof v}`);
373
+ }
374
+ if (customNames.has(v)) {
375
+ out.add(v);
376
+ continue;
377
+ }
378
+ const name = canonicalType(v);
379
+ if (!KNOWN_TYPES.has(name)) {
380
+ throw new Error(`unknown entity type ${JSON.stringify(v)} in ${param}. Known types: ` +
381
+ [...KNOWN_TYPES].sort().join(", ") +
382
+ (customNames.size ? `; custom patterns: ${[...customNames].sort().join(", ")}` : ""));
383
+ }
384
+ out.add(name);
385
+ }
386
+ return out;
387
+ }
388
+ /**
389
+ * Split off the detections of the types the caller excluded. Applied to each
390
+ * span's final type, after ranking: dropping an excluded type's candidates
391
+ * earlier would let the next-best type claim the same span.
392
+ */
393
+ function applyTypeExclusion(text, detections, excluded) {
394
+ if (excluded.size === 0)
395
+ return [detections, []];
396
+ const kept = [];
397
+ const exempted = [];
398
+ for (const d of detections) {
399
+ const name = String(d.entityType);
400
+ if (excluded.has(name)) {
401
+ exempted.push({
402
+ entityType: d.entityType, start: d.start, end: d.end,
403
+ text: text.slice(d.start, d.end), rule: name, ruleKind: "type",
404
+ });
405
+ }
406
+ else
407
+ kept.push(d);
408
+ }
409
+ return [kept, exempted];
410
+ }
411
+ /** Allowlist first, then excluded types; every exemption in text order. */
412
+ function exempt(text, detections, allowed, domains, excluded) {
413
+ const [afterValues, byValue] = applyAllowlist(text, detections, allowed, domains);
414
+ const [kept, byType] = applyTypeExclusion(text, afterValues, excluded);
415
+ const exempted = [...byValue, ...byType];
416
+ if (byValue.length && byType.length)
417
+ exempted.sort((a, b) => a.start - b.start || b.end - a.end);
418
+ return [kept, exempted];
419
+ }
353
420
  /** Two label schemes for the same spans cannot both apply. */
354
421
  function checkLabelOptions(options) {
355
422
  if (options.tokenize && options.referentialIntegrity) {
@@ -430,19 +497,26 @@ export class EuRedact {
430
497
  maxInputLength;
431
498
  allowlist;
432
499
  allowlistDomains;
500
+ customNames = new Set();
501
+ excludeTypes;
433
502
  /**
434
503
  * @param options.maxInputLength Longest document `redact` accepts, in characters.
435
504
  * @param options.allowlist Values never to redact, for every call on this
436
505
  * instance — an organisation's own name, its own addresses. Merged with
437
506
  * the per-call `allowlist`. Matched whole, case-insensitively.
507
+ * @param options.excludeTypes Entity types never reported or masked, for
508
+ * every call on this instance, e.g. `["JOB_TITLE", "FINANCIAL_AMOUNT"]`.
509
+ * Merged with the per-call value. Unknown names throw.
438
510
  */
439
511
  constructor(options) {
440
512
  this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
441
513
  this.allowlist = normalizeAllowlist(options?.allowlist);
442
514
  this.allowlistDomains = normalizeDomains(options?.allowlistDomains);
515
+ this.excludeTypes = normalizeExcludeTypes(options?.excludeTypes);
443
516
  }
444
517
  addCustomPattern(name, pattern) {
445
518
  this.engine.addCustomPattern(name, pattern);
519
+ this.customNames.add(name);
446
520
  this.cache.clear();
447
521
  }
448
522
  /** Clear the result cache and referential integrity mappings.
@@ -458,6 +532,13 @@ export class EuRedact {
458
532
  allowed.set(k, v);
459
533
  return allowed;
460
534
  }
535
+ /** The instance's excluded types merged with a call's. */
536
+ excludedFor(options) {
537
+ const excluded = new Set(this.excludeTypes);
538
+ for (const t of normalizeExcludeTypes(options.excludeTypes, this.customNames))
539
+ excluded.add(t);
540
+ return excluded;
541
+ }
461
542
  /** The instance domain list merged with a call's. */
462
543
  domainsFor(options) {
463
544
  const domains = new Map(this.allowlistDomains);
@@ -514,22 +595,26 @@ export class EuRedact {
514
595
  }
515
596
  const allowed = this.allowedFor(options);
516
597
  const domains = this.domainsFor(options);
598
+ const excluded = this.excludedFor(options);
517
599
  const client = new CloudClient();
518
600
  // Dates are always on, whatever detectDates says: that is the rules output
519
601
  // the model was trained against, and a date of birth the rules can place
520
602
  // has no reason to travel. It is the one ignored option that is safe to
521
603
  // ignore — it can only cause MORE to be detected, never less.
522
604
  //
523
- // No allowlist and no tokens on this pass. An exemption says what the
524
- // caller wants back, not what may leave; both are applied below, after the
605
+ // No allowlist, no excluded types and no tokens on this pass. An exemption
606
+ // says what the caller wants back, not what may leave: an excluded EMAIL is
607
+ // still masked in the request. All three are applied below, after the
525
608
  // response, to the merged spans.
526
609
  const local = this.redactRules(text, { countries, detectDates: true, cache: options.cache }, new Map(), new Map());
527
610
  const [masked, labels] = maskForCloud(text, local.detections);
528
611
  // The client converts the service's code-point offsets to UTF-16 units of
529
612
  // the text it sent, so everything from here on is in one unit.
530
- const remote = await client.redact(masked, { country: countries[0] });
613
+ const remote = await client.redact(masked, {
614
+ country: countries[0], idempotencyKey: options.idempotencyKey,
615
+ });
531
616
  // A new array throughout: `local` may be the cached rules result.
532
- const [detections, exempted] = applyAllowlist(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains);
617
+ const [detections, exempted] = exempt(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains, excluded);
533
618
  detections.sort((a, b) => a.start - b.start || b.end - a.end);
534
619
  const tokenMapper = options.tokenize ? new TokenMapper(text, detections) : null;
535
620
  return {
@@ -539,6 +624,7 @@ export class EuRedact {
539
624
  source: "cloud",
540
625
  tokens: tokenMapper ? tokenMapper.tokens : {},
541
626
  exempted,
627
+ ...(remote.cloud ? { cloud: remote.cloud } : {}),
542
628
  };
543
629
  }
544
630
  redact(text, options = {}) {
@@ -547,6 +633,7 @@ export class EuRedact {
547
633
  checkLabelOptions(options);
548
634
  const allowed = this.allowedFor(options);
549
635
  const domains = this.domainsFor(options);
636
+ const excluded = this.excludedFor(options);
550
637
  const requestedMode = options.mode ?? "rules";
551
638
  if (requestedMode === "cloud") {
552
639
  // Cannot be served synchronously: the cloud tier is a network call, and
@@ -560,10 +647,13 @@ export class EuRedact {
560
647
  if (requestedMode !== "rules") {
561
648
  throw new Error(`unknown mode ${JSON.stringify(requestedMode)}: expected "rules" or "cloud"`);
562
649
  }
563
- return this.redactRules(text, options, allowed, domains);
650
+ if (options.idempotencyKey !== undefined) {
651
+ throw new Error('idempotencyKey applies to mode: "cloud" only: a rules-only redaction makes no request to deduplicate');
652
+ }
653
+ return this.redactRules(text, options, allowed, domains, excluded);
564
654
  }
565
655
  /** The local pipeline: normalise, detect, exempt, mask. */
566
- redactRules(text, options, allowed, domains) {
656
+ redactRules(text, options, allowed, domains, excluded = new Set()) {
567
657
  if (text.length > this.maxInputLength) {
568
658
  throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
569
659
  `(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
@@ -586,7 +676,8 @@ export class EuRedact {
586
676
  // JSON rather than a joined string: an entry may itself contain the
587
677
  // separator, and two different lists must never share a key.
588
678
  const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
589
- + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "");
679
+ + (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "")
680
+ + (excluded.size ? "|excl=" + JSON.stringify([...excluded].sort()) : "");
590
681
  const cacheMode = `rules|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
591
682
  let cacheKey;
592
683
  if (cache) {
@@ -610,7 +701,7 @@ export class EuRedact {
610
701
  detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
611
702
  }
612
703
  let exempted;
613
- [detections, exempted] = applyAllowlist(text, detections, allowed, domains);
704
+ [detections, exempted] = exempt(text, detections, allowed, domains, excluded);
614
705
  detections.sort((a, b) => a.start - b.start || b.end - a.end);
615
706
  const tokenMapper = tokenize ? new TokenMapper(text, detections) : null;
616
707
  const redacted = applyReplacements(text, detections, this.labelFor(referentialIntegrity, tokenMapper));
@@ -69,7 +69,8 @@ export declare enum EntityType {
69
69
  BIOMETRIC_REF = "BIOMETRIC_REF",
70
70
  FINANCIAL_AMOUNT = "FINANCIAL_AMOUNT",
71
71
  QUASI_IDENTIFIER = "QUASI_IDENTIFIER",
72
- CREDENTIAL = "CREDENTIAL",
72
+ /** Legacy alias of SECRET: every credential, the model's included, is SECRET (rules-engine#111). */
73
+ CREDENTIAL = "SECRET",
73
74
  URL = "URL",
74
75
  OTHER = "OTHER"
75
76
  }
@@ -116,7 +117,13 @@ export interface Detection {
116
117
  text: string;
117
118
  source: DetectionSource;
118
119
  country: string | null;
119
- confidence: string;
120
+ /**
121
+ * How the type was arrived at: "high", "medium" or "low" for a rules
122
+ * detection. `null` for a span the cloud model found and reported no
123
+ * confidence for: the model emits none today, and "high" would claim the
124
+ * checksum-backed evidence a rules detection has (rules-engine#108).
125
+ */
126
+ confidence: string | null;
120
127
  /**
121
128
  * How strongly the document supports {@link Detection.country}, in [0, 1].
122
129
  *
@@ -134,7 +141,7 @@ export interface Detection {
134
141
  outOfScope?: boolean;
135
142
  }
136
143
  /**
137
- * A detection the allowlist kept out of the redacted output.
144
+ * A detection the allowlist or `excludeTypes` kept out of the redacted output.
138
145
  *
139
146
  * An exemption is a deliberate decision to leave a direct identifier in a
140
147
  * document that otherwise claims to be redacted, so it is reported rather than
@@ -146,10 +153,58 @@ export interface Exemption {
146
153
  start: number;
147
154
  end: number;
148
155
  text: string;
149
- /** The allowlist entry that matched, as the caller wrote it. */
156
+ /** The allowlist entry that matched, as the caller wrote it; for an
157
+ * excluded type, the type's canonical name. */
150
158
  rule: string;
151
- /** `"value"` for the literal allowlist, `"domain"` for allowlistDomains. */
152
- ruleKind: "value" | "domain";
159
+ /** `"value"` for the literal allowlist, `"domain"` for allowlistDomains,
160
+ * `"type"` for excludeTypes (rules-engine#125). */
161
+ ruleKind: "value" | "domain" | "type";
162
+ }
163
+ /**
164
+ * One reason a cloud request cost what it did. `code` is one of the service's
165
+ * factor codes (`rules_only`, `document_length`, `instructions`,
166
+ * `dense_document`, `special_category_check`, `long_document`, `batch_rate`),
167
+ * kept as a string so a code added later on the service still arrives.
168
+ */
169
+ export interface UsageFactor {
170
+ code: string;
171
+ /** One sentence explaining the factor, written by the service. */
172
+ detail: string;
173
+ /** `special_category_check` only: the kinds of data that triggered it. */
174
+ types?: string[];
175
+ }
176
+ /**
177
+ * What a cloud request cost, and why (rules-engine#89). A total and its
178
+ * reasons, never a cost per step; `credits` is `round(tokens * billingRate)`,
179
+ * the formula the service debits by.
180
+ */
181
+ export interface Usage {
182
+ tokens: number;
183
+ billingRate: number;
184
+ credits: number;
185
+ factors: UsageFactor[];
186
+ }
187
+ /**
188
+ * Something the model found that could not be placed in the document: the
189
+ * service could not match it to a position in what it was sent, so nothing was
190
+ * masked for it. Reported so a caller can decide, rather than never being told.
191
+ */
192
+ export interface Unlocated {
193
+ text: string;
194
+ entityType: EntityType | string;
195
+ }
196
+ /** What the cloud service said about a request, beside its spans (rules-engine#89). */
197
+ export interface CloudInfo {
198
+ /** The service's job id: for support, for `Jobs.retrieve`, and to recognise an
199
+ * idempotent replay. `null` for a batch document. */
200
+ jobId: string | null;
201
+ /** The public name of the model that answered, `null` when only rules ran. */
202
+ modelVersion: string | null;
203
+ /** What the request cost and why. Absent when the service does not report it
204
+ * (older versions, batch documents) or reported it malformed. */
205
+ usage?: Usage;
206
+ /** What the model found but could not place in the text it was sent. */
207
+ unlocated: Unlocated[];
153
208
  }
154
209
  export interface RedactResult {
155
210
  redactedText: string;
@@ -188,6 +243,9 @@ export interface RedactResult {
188
243
  /** Detections the allowlist kept out of the output, with the rule that
189
244
  * matched. Empty unless an allowlist was in force. */
190
245
  exempted: Exemption[];
246
+ /** What the cloud service reported beside the spans: job id, model, cost and
247
+ * anything it could not place. Absent on a rules-only result. */
248
+ cloud?: CloudInfo;
191
249
  }
192
250
  export interface PatternDef {
193
251
  entityType: EntityType | string;
package/dist/esm/types.js CHANGED
@@ -75,7 +75,8 @@ export var EntityType;
75
75
  EntityType["BIOMETRIC_REF"] = "BIOMETRIC_REF";
76
76
  EntityType["FINANCIAL_AMOUNT"] = "FINANCIAL_AMOUNT";
77
77
  EntityType["QUASI_IDENTIFIER"] = "QUASI_IDENTIFIER";
78
- EntityType["CREDENTIAL"] = "CREDENTIAL";
78
+ /** Legacy alias of SECRET: every credential, the model's included, is SECRET (rules-engine#111). */
79
+ EntityType["CREDENTIAL"] = "SECRET";
79
80
  EntityType["URL"] = "URL";
80
81
  EntityType["OTHER"] = "OTHER";
81
82
  })(EntityType || (EntityType = {}));
@@ -89,6 +90,8 @@ export const LEGACY_TYPE_ALIASES = {
89
90
  NAME: "PERSON_NAME",
90
91
  STREET_ADDRESS: "ADDRESS",
91
92
  NATIONALITY_ETHNICITY: "SENSITIVE_ATTRIBUTE",
93
+ // The model labels prose credentials CREDENTIAL; the SDK reports every credential as SECRET (rules-engine#111).
94
+ CREDENTIAL: "SECRET",
92
95
  };
93
96
  /**
94
97
  * Resolve a possibly-legacy type name to its canonical form.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "euredact",
3
- "version": "0.6.0",
3
+ "version": "0.7.1",
4
4
  "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
5
5
  "homepage": "https://euredact.dev",
6
6
  "license": "Apache-2.0",
@@ -20,10 +20,15 @@
20
20
  }
21
21
  }
22
22
  },
23
+ "browser": {
24
+ "./dist/esm/platform.js": "./dist/esm/platform.browser.js",
25
+ "./dist/cjs/platform.js": "./dist/cjs/platform.browser.js"
26
+ },
23
27
  "files": [
24
28
  "dist",
25
29
  "README.md",
26
- "LICENSE"
30
+ "LICENSE",
31
+ "NOTICE"
27
32
  ],
28
33
  "engines": {
29
34
  "node": ">=16.0.0"
@@ -54,7 +59,7 @@
54
59
  "build:cjs": "tsc -p tsconfig.cjs.json",
55
60
  "postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
56
61
  "build": "rm -rf dist && npm run build:esm && npm run build:cjs",
57
- "test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:tokenize && npm run test:allowlist && npm run test:eval-scoring && npm run test:conformance",
62
+ "test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:cloud-api && npm run test:tokenize && npm run test:allowlist && npm run test:exclude-types && npm run test:eval-scoring && npm run test:platform && npm run test:batches && npm run test:conformance",
58
63
  "benchmark": "node --import tsx src/__tests__/benchmark.ts",
59
64
  "prepublishOnly": "npm run build",
60
65
  "test:spec": "node --import tsx src/__tests__/spec.ts",
@@ -64,9 +69,13 @@
64
69
  "test:inference": "node --import tsx src/__tests__/inference.ts",
65
70
  "dump": "node --import tsx src/__tests__/dump.ts",
66
71
  "test:cloud": "node --import tsx src/__tests__/cloud.ts",
72
+ "test:cloud-api": "node --import tsx src/__tests__/cloudApi.ts",
67
73
  "test:tokenize": "node --import tsx src/__tests__/tokenize.ts",
68
74
  "test:allowlist": "node --import tsx src/__tests__/allowlist.ts",
69
- "test:eval-scoring": "node --import tsx src/__tests__/evalScoring.ts"
75
+ "test:exclude-types": "node --import tsx src/__tests__/excludeTypes.ts",
76
+ "test:eval-scoring": "node --import tsx src/__tests__/evalScoring.ts",
77
+ "test:platform": "node --import tsx src/__tests__/platform.ts",
78
+ "test:batches": "node --import tsx src/__tests__/batches.ts"
70
79
  },
71
80
  "devDependencies": {
72
81
  "@types/node": "22.10.2",