euredact 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -82,7 +82,8 @@ console.log(result.detections);
82
82
  | keep relationships visible across a whole session | `{ referentialIntegrity: true }` |
83
83
  | never redact my own company name or addresses | `{ allowlist: [...] }`, or `new EuRedact({ allowlist })` |
84
84
  | never redact anything at my own domain | `{ allowlistDomains: ["acme.be"] }` |
85
- | audit what the allowlist kept in the document | `result.exempted` |
85
+ | leave whole types in the text (job titles, amounts) | `{ excludeTypes: ["JOB_TITLE", "FINANCIAL_AMOUNT"] }`, or `new EuRedact({ excludeTypes })` — see [Excluding entity types](#excluding-entity-types) |
86
+ | audit what the allowlist or `excludeTypes` kept in the document | `result.exempted` |
86
87
  | catch person names, employers, job titles, diagnoses | `redactAsync(text, { mode: "cloud" })` — see [Cloud tier](#cloud-tier) |
87
88
  | include dates of birth | `{ detectDates: true }` |
88
89
  | redact a document too large for one call | `context` + `chunkOffset` — see [Chunked documents](#chunked-documents) |
@@ -112,6 +113,7 @@ interface RedactOptions {
112
113
  tokenize?: boolean; // reversible tokens, EMAIL_K7Q2
113
114
  allowlist?: string[] | null; // values never redacted
114
115
  allowlistDomains?: string[] | null; // domains never redacted (EMAIL/URL)
116
+ excludeTypes?: string[] | null; // entity types neither reported nor masked
115
117
  detectDates?: boolean; // include DOB / date of death
116
118
  cache?: boolean; // reuse results for identical input
117
119
  idempotencyKey?: string; // mode: "cloud" only; see the table below
@@ -136,6 +138,7 @@ interface RedactOptions {
136
138
  | `tokenize` | `false` | Reversible token per value (`EMAIL_K7Q2`), with the mapping in `result.tokens` for [`restore()`](#restoretext-tokens). Unique to the **call**. Cannot be combined with `referentialIntegrity`. See [Reversible tokenization](#reversible-tokenization). |
137
139
  | `allowlist` | `null` | Values never to redact, whole-span and case-insensitive, merged with the instance's list. Structured identifiers (IBAN, phone, VAT, …) also match across spacing and hyphenation. Applies to cloud-tier types too. See [Allowlist](#allowlist). |
138
140
  | `allowlistDomains` | `null` | Domains whose addresses are never redacted, e.g. `["acme.be"]`. Applies to `EMAIL` and `URL` only, and covers subdomains. See [Allowlist](#allowlist). |
141
+ | `excludeTypes` | `null` | Entity types neither reported nor masked, e.g. `["JOB_TITLE"]`; merged with the instance's list. Unknown names throw. See [Excluding entity types](#excluding-entity-types). |
139
142
 
140
143
  **Long documents and tiers**
141
144
 
@@ -229,6 +232,8 @@ The `EuRedact` class exposes `redact()`, `redactAsync()`, `redactBatch()`,
229
232
  new EuRedact({
230
233
  maxInputLength?: number, // default 10_485_760 (~10 MB)
231
234
  allowlist?: string[] | null,
235
+ allowlistDomains?: string[] | null,
236
+ excludeTypes?: string[] | null,
232
237
  })
233
238
  ```
234
239
 
@@ -237,6 +242,7 @@ new EuRedact({
237
242
  | `maxInputLength` | `10_485_760` | Longest document `redact()` accepts, in characters. Above it, it throws — split the input or raise the ceiling. |
238
243
  | `allowlist` | `null` | Values never to redact, for every call on this instance. Merged with the per-call `allowlist`. See [Allowlist](#allowlist). |
239
244
  | `allowlistDomains` | `null` | Domains never to redact, for every call on this instance. |
245
+ | `excludeTypes` | `null` | Entity types never reported or masked, for every call on this instance. See [Excluding entity types](#excluding-entity-types). |
240
246
 
241
247
  The result cache, referential-integrity labels and custom patterns are all per
242
248
  instance, which is what makes one instance per tenant the right default.
@@ -259,7 +265,7 @@ interface RedactResult {
259
265
  detectionMode: string; // "declared" if countries was passed,
260
266
  // "inferred" otherwise
261
267
  tokens: Record<string, string>; // token -> original value; only with tokenize: true
262
- exempted: Exemption[]; // spans the allowlist kept, with the rule that matched
268
+ exempted: Exemption[]; // spans the allowlist or excludeTypes kept, with the rule
263
269
  cloud?: CloudInfo; // mode: "cloud" only: job id, model, cost, unplaced
264
270
  // findings — see "What the service said"
265
271
  }
@@ -1409,8 +1415,8 @@ sdk.redact("Mail jan@acme.be or piet@acme.be", { countries: ["NL"] }).redactedTe
1409
1415
  label boundary, so `acme.be` does **not** exempt `evilacme.be`. Entries may be
1410
1416
  written `acme.be`, `@acme.be` or `.acme.be`.
1411
1417
 
1412
- There are deliberately **no wildcards**. The allowlist is the only option that
1413
- turns redaction *off*, so an over-broad entry fails toward under-redaction and
1418
+ There are deliberately **no wildcards**. The allowlist and `excludeTypes` are
1419
+ the only options that turn redaction *off*, so an over-broad entry fails toward under-redaction and
1414
1420
  does so silently.
1415
1421
 
1416
1422
  ### What was exempted
@@ -1422,10 +1428,51 @@ for (const e of r.exempted) console.log(e.entityType, e.text, e.rule, e.ruleKind
1422
1428
  ```
1423
1429
 
1424
1430
  `Exemption` carries `entityType`, `start`, `end`, `text`, the `rule` that
1425
- matched as you wrote it, and `ruleKind` (`"value"` or `"domain"`). An exempted
1431
+ matched as you wrote it, and `ruleKind` (`"value"`, `"domain"` or `"type"`). An exempted
1426
1432
  span is absent from `detections`, so `exempted` is the only record that it was
1427
1433
  found at all.
1428
1434
 
1435
+ ## Excluding entity types
1436
+
1437
+ A deployment whose taxonomy is narrower than euRedact's can switch whole types
1438
+ off instead of filtering every result itself. Excluded types are neither in
1439
+ `detections` nor masked:
1440
+
1441
+ ```ts
1442
+ const r = redact("Mail anna@acme.be or call +32 470 12 34 56.", {
1443
+ countries: ["BE"], excludeTypes: ["PHONE"],
1444
+ });
1445
+ r.redactedText;
1446
+ // 'Mail [EMAIL] or call +32 470 12 34 56.'
1447
+ for (const e of r.exempted) console.log(e.entityType, e.text, e.rule, e.ruleKind);
1448
+ // PHONE +32 470 12 34 56 PHONE type
1449
+ ```
1450
+
1451
+ Set it once for a deployment with `new EuRedact({ excludeTypes: [...] })`; a per-call
1452
+ list is merged with it.
1453
+
1454
+ - **Nothing is excluded by default.** Every type is detected and redacted
1455
+ unless you name it.
1456
+ - **Names are checked.** They are case-insensitive, and legacy names resolve
1457
+ (`IBAN` is `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A registered custom
1458
+ pattern's name works too. An unknown name throws, so a typo
1459
+ cannot silently switch nothing off.
1460
+ - **It applies to each span's final type.** In
1461
+ `NHS Number: 5556953627` the digits also pass the Austrian SVNr check.
1462
+ Excluding `HEALTH_INSURANCE` leaves them in the text; it does not let them
1463
+ be masked as an `SSN` instead.
1464
+ - **Every excluded detection is recorded** in `result.exempted`, with
1465
+ `ruleKind: "type"` and the type's name as `rule`, so an audit can see what
1466
+ was deliberately left in.
1467
+ - **In `mode: "cloud"`** an excluded rules type is still masked in the text
1468
+ sent to the service. The option decides what comes back, not what may leave
1469
+ your machine. Model types (`JOB_TITLE`, `FINANCIAL_AMOUNT`, …) are filtered
1470
+ from the merged result.
1471
+ - **There is no include-list.** A list of types to keep would leave every type
1472
+ a later release adds unmasked for you, without notice.
1473
+ - **A type is one switch.** `INTERNAL_ID` covers employee and customer numbers
1474
+ as well as ticket and case numbers; excluding it leaves all of them in.
1475
+
1429
1476
  ## Architecture
1430
1477
 
1431
1478
  ```
@@ -4,6 +4,11 @@ exports.COUNTRY_CONFIGS = void 0;
4
4
  const types_js_1 = require("../../types.js");
5
5
  const validators_js_1 = require("../validators.js");
6
6
  const suppressors_js_1 = require("../suppressors.js");
7
+ /** An assigned value: not opened by a quote or bracket, eight characters or
8
+ * more, not ended by punctuation or a closer (rules-engine#35, #126). */
9
+ const ASSIGNED = String.raw `[^\s"'\x60\[\]{}()<>„“”«»‚‘’]\S{6,}[^\s.,;:!?)'"\]}\x60“”»‘’]`;
10
+ /** A quoted assigned value: exactly what the quotes hold (rules-engine#126). */
11
+ const QUOTED_ASSIGNED = ['"', "'", "\x60"].flatMap((q) => [" ", ""].map((gap) => String.raw `(?<=[:=]` + gap + q + String.raw `)[^\s` + q + String.raw `]{8,}(?=` + q + ")")).join("|");
7
12
  /** Day, month as a word, year ("4. März 1948"), behind a date label (rules-engine#115). */
8
13
  function writtenDatePattern() {
9
14
  const names = new Set(suppressors_js_1.MONTH_NAMES.split("|").filter(Boolean));
@@ -146,6 +151,7 @@ const DOB_CONTEXT = [
146
151
  "γεννηθείς", "γεννήθηκε", "ημερομηνία γέννησης",
147
152
  // West Slavic.
148
153
  "urodzony", "urodzona", "data urodzenia",
154
+ "ur.",
149
155
  "narozen", "datum narození", "narodený", "dátum narodenia",
150
156
  // Hungarian, Romanian, Bulgarian.
151
157
  "született", "születési", "născut", "născută", "data nașterii",
@@ -268,14 +274,20 @@ const SHARED = {
268
274
  // AWS Secret Access Key (40 chars, base64 with +/ after an AKIA line)
269
275
  p(types_js_1.EntityType.SECRET, String.raw `(?<=aws_secret_access_key\s{0,4}[:=]\s{0,4})[A-Za-z0-9+/]{40}`, null, "AWS Secret Access Key"),
270
276
  // --- Secret / API Key (connection strings with embedded credentials) ---
271
- p(types_js_1.EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]{1,256}:[^\s@]{1,256}@[^\s]+`, null, "Connection string with credentials"),
277
+ // The tail stops at a quote (rules-engine#126).
278
+ p(types_js_1.EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]{1,256}:[^\s@]{1,256}@[^\s"'\x60]+`, null, "Connection string with credentials"),
272
279
  // --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
273
280
  // The final character may not be sentence punctuation. `\S` does not stop
274
281
  // at a full stop, so "Reisepass: CA1234567." was claimed as the
275
282
  // ten-character span "CA1234567." -- a passport number plus the sentence's
276
283
  // period, masked as a credential because a "credentials" word sat within
277
284
  // the context window (rules-engine#35).
278
- p(types_js_1.EntityType.SECRET, String.raw `(?<=[:=] )\S{7,}[^\s.,;:!?)'"]|(?<=[:=])\S{7,}[^\s.,;:!?)'"]`, "high_entropy", "Assigned secret value", SECRET_CONTEXT, true),
285
+ // The value starts after an opening quote or bracket and ends before its
286
+ // closer: `user = "svc_terraform"` was the span `"svc_terraform`
287
+ // (rules-engine#126).
288
+ // A quoted value is what the quotes hold, closing punctuation included.
289
+ p(types_js_1.EntityType.SECRET, QUOTED_ASSIGNED + "|" + [String.raw `(?<=[:=] )`, String.raw `(?<=[:=])`, String.raw `(?<=[:=] ["'\x60\[{(])`, String.raw `(?<=[:=]["'\x60\[{(])`]
290
+ .map((lb) => lb + ASSIGNED).join("|"), "high_entropy", "Assigned secret value", SECRET_CONTEXT, true),
279
291
  // --- Secret / API Key (entropy-based fallback for longer tokens) ---
280
292
  // Only the *trailing* \b is replaced. It could not match when the token run
281
293
  // ended on "-", "+" or "/", so the engine backtracked across two
@@ -555,7 +567,10 @@ const PT = {
555
567
  p(types_js_1.EntityType.PHONE, String.raw `\b[29]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}\b`),
556
568
  p(types_js_1.EntityType.PHONE, String.raw `\b[29]\d{8}\b`),
557
569
  p(types_js_1.EntityType.PHONE, String.raw `\+351\s?[29]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}`),
558
- p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b[A-Z0-9]{2}-[A-Z0-9]{2}-[A-Z0-9]{2}\b`),
570
+ // The four formats Portugal has issued (rules-engine#126).
571
+ p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b(?:\d{2}-\d{2}-[A-Z]{2}|\d{2}-[A-Z]{2}-\d{2}|[A-Z]{2}-\d{2}-\d{2}|[A-Z]{2}-\d{2}-[A-Z]{2})\b`),
572
+ // Any three pairs, behind a plate label (rules-engine#126).
573
+ p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b[A-Z0-9]{2}-[A-Z0-9]{2}-[A-Z0-9]{2}\b`, null, "Portuguese license plate — AA-XX-XX, behind its label", ["matrícula", "matricula", "placa", "license plate", "number plate", "plate", "Kennzeichen", "kenteken", "immatriculation", "targa"], true),
559
574
  p(types_js_1.EntityType.POSTAL_CODE, String.raw `\b\d{4}-\d{3}\b`),
560
575
  ],
561
576
  };
@@ -630,6 +645,8 @@ const UK = {
630
645
  code: "UK",
631
646
  name: "United Kingdom",
632
647
  patterns: [
648
+ // A sort code behind its label; no longer found through the Portuguese plate shape (rules-engine#126).
649
+ p(types_js_1.EntityType.BANK_ACCOUNT, String.raw `\b\d{2}-\d{2}-\d{2}\b`, null, "UK sort code — NN-NN-NN, behind its label", ["sort code", "sortcode", "sort-code"], true),
633
650
  // Healthcare-provider numbers, behind their label: no shape of their own, so without a pattern nothing claimed them, or a phone or a foreign checksum did (canon HEALTHCARE_PROVIDER; rules-engine#111).
634
651
  p(types_js_1.EntityType.HEALTHCARE_PROVIDER, String.raw `\b\d{7}\b`, null, "UK GMC number — 7 digits", ["GMC number", "GMC no", "GMC reference", "GMC reg", "GMC:", "GMC-Nr", "General Medical Council"], true),
635
652
  // UK NINO: a social-security number, so SSN, as the canon defaults it (rules-engine#47).
@@ -203,6 +203,27 @@ function suppressSecretOverStructured(text, match) {
203
203
  * natural word. A random all-letter token would be mixed case and is left alone
204
204
  * — losing a real secret is the expensive direction.
205
205
  */
206
+ /**
207
+ * Values an assigned-secret rule finds after a "=" or ":" that are names, not
208
+ * secrets (rules-engine#126): `${VAR}` references, identifiers, hyphenated
209
+ * compounds, image tags, product versions, code calls. Not paths, host names
210
+ * or reference numbers: those carry personal data ("mac-mhartmann88").
211
+ * Kept in sync with the Python SDK's _SECRET_NAME_SHAPES.
212
+ */
213
+ const SECRET_NAME_SHAPES = new RegExp([
214
+ String.raw `^\$\{[A-Za-z_][A-Za-z0-9_]*(?::?[-=?+][^}]*)?\}?$`,
215
+ String.raw `^[a-z]{2,}(?:_[a-z]{2,})+$`, String.raw `^[A-Z]{2,}(?:_[A-Z]{2,})+$`,
216
+ String.raw `^(?:[A-Z]+[a-z]{2,}){3,}(?:V?\d{1,2})?$`,
217
+ String.raw `^[a-z][a-z0-9._/-]*:(?:\d+(?:\.\d+)*(?:-[a-z0-9]+)*|latest|stable|alpine|slim)$`,
218
+ String.raw `^[A-Za-z][\w.-]*\/\d+(?:\.\d+)+$`,
219
+ String.raw `^[\w.]+\(["'\x60]`,
220
+ ].join("|"));
221
+ /** A hyphenated compound of words and acronyms; two capitalised words alone
222
+ * ("Ann-Sophie", "Caron-Cordier") are names and are left alone. */
223
+ const COMPOUND_PART = String.raw `(?:[A-ZÀ-Þ]{2,6}\d{0,3}|[A-Za-zÀ-ÿ][a-zß-ÿ]{2,})`;
224
+ const COMPOUND = new RegExp(String.raw `^` + COMPOUND_PART + String.raw `(?:-` + COMPOUND_PART + String.raw `)+$`);
225
+ const NAME_PAIR = /^[A-ZÀ-Þ][a-zß-ÿ]{2,}-[A-ZÀ-Þ][a-zß-ÿ]{2,}$/;
226
+ const CONNECTION_WITH_REFERENCE = /^[a-z]+:\/\/[^\s:@]+:\$\{[^}]*\}@/;
206
227
  function suppressSecretNotASecret(_text, match) {
207
228
  if (match.patternDef.entityType !== types_js_1.EntityType.SECRET)
208
229
  return false;
@@ -217,6 +238,10 @@ function suppressSecretNotASecret(_text, match) {
217
238
  return true;
218
239
  if (HYPHENATED_WORDS.test(bare))
219
240
  return true;
241
+ if (SECRET_NAME_SHAPES.test(bare) || CONNECTION_WITH_REFERENCE.test(bare))
242
+ return true;
243
+ if (COMPOUND.test(bare) && !NAME_PAIR.test(bare))
244
+ return true;
220
245
  const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
221
246
  const oneWord = bare === bare.toLowerCase() ||
222
247
  (bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
@@ -358,7 +383,20 @@ function suppressSequential(_text, match) {
358
383
  // the "]". The opening bracket is still required.
359
384
  const PLACEHOLDER = /^(?:\[([A-Z][A-Z0-9_]*)\]?|([A-Z][A-Z0-9_]*?)_(?:\d+|[A-HJ-NP-Z2-9]{4}))$/;
360
385
  const KNOWN_TYPE_NAMES = new Set(Object.values(types_js_1.EntityType));
361
- function suppressRedactionPlaceholder(_text, match) {
386
+ function suppressRedactionPlaceholder(text, match) {
387
+ // The name between the brackets: an assigned value now starts after an
388
+ // opening bracket, so "credentials: [POSTAL_CODE]" offers "POSTAL_CODE" alone
389
+ // (rules-engine#33, #126).
390
+ if (text[match.start - 1] === "[" && text[match.end] === "]" && KNOWN_TYPE_NAMES.has(match.text))
391
+ return true;
392
+ // A placeholder inside the span: on text masked once, a value running across
393
+ // "[IP_ADDRESS]" is two things joined (rules-engine#126).
394
+ // One character either side: a span may start just inside the bracket.
395
+ const around = text.slice(Math.max(0, match.start - 1), match.end + 1);
396
+ for (const m of around.matchAll(/\[([A-Z][A-Z0-9_]*)\]/g)) {
397
+ if (KNOWN_TYPE_NAMES.has(m[1]))
398
+ return true;
399
+ }
362
400
  const found = PLACEHOLDER.exec(match.text);
363
401
  if (found === null)
364
402
  return false;
@@ -506,6 +544,33 @@ const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|d
506
544
  */
507
545
  const COUNT_LABEL_BEFORE = /(?<![A-Za-z0-9_])(?:employees|headcount|staff|mitarbeiter(?:innen)?|beschäftigte|werknemers|medewerkers|employés|salariés|effectifs?|dipendenti|empleados|pracowników|zaměstnanců)\s*[:=]?\s*$/i;
508
546
  const NOT_POSTAL_BEFORE = /(?<![A-Za-z0-9_])(?:(?:ISO(?:\/IEC)?|IEC|EN|DIN(?:\s+EN)?|NEN(?:-EN)?|NF\s+EN|BS(?:\s+EN)?|PN-EN|ÖNORM)[\s\-]*|[A-Za-z]*[a-z][A-Z][A-Za-z]*\s+)$/;
547
+ /**
548
+ * What else a number is, by what touches it (rules-engine#126): a build,
549
+ * version, event, lease, policy or ticket number; a port; an invoice, reference
550
+ * or study-subject number; a process id; a JSON id or port field; an HTTP
551
+ * response size; the hour after a date; a lab reference range; or a quantity
552
+ * with its unit after it. Kept in sync with the Python SDK.
553
+ */
554
+ const TECH_WORD_BEFORE = /(?<![A-Za-z0-9_])(?:build|version|policy\s+id|ticket(?:\s+[A-Za-z]{2,10})?|event|lease|pid|job|port|rfc|facture|invoice|r[ée]f[ée]rence|n[°º]|tn|sae|uczestnik|subject|narratief|participant)\.?\s*[:#=]?\s*["(«[]?\s*$/i;
555
+ const TECH_SHAPE_BEFORE = new RegExp("(?:" + [
556
+ String.raw `"[A-Za-z_]*(?:id|port)"\s*:\s*"?`, // "parent_id": 4400
557
+ String.raw `(?<![A-Za-z0-9])[A-Z_]*PORT\s*[:=]\s*"?`, // DB_PORT: "5432"
558
+ String.raw `(?:localhost|[a-z0-9\-]+\.[a-z0-9.\-]+):`, // localhost:3000
559
+ String.raw `(?<![\d.])\d{2,5}:`, // "8080:8080"
560
+ String.raw `HTTP\/\d(?:\.\d)?"?\s+\d{3}\s+`, // HTTP/1.1" 200 2048
561
+ String.raw `\d+\.\d+(?:\.\d+)+\s*\(`, // build 2.4.0 (4012)
562
+ String.raw `(?<!\d)\d{1,2}[./]\d{1,2}[./]`, // 03.10.2026 23:50
563
+ String.raw `\((?:N|norm(?:al)?|ref)\s*:?\s*`, // (N : 30-400)
564
+ String.raw `\|`,
565
+ ].join("|") + ")$", "i");
566
+ const POSTAL_UNIT_AFTER = /^(?:\s?(?:ms|s|sec|secs|seconds?|secondes?|sekunden|sek|min|bits?|bytes?|kb|mb|gb|tb|kbps|mbps|px|%|records|rows|entries|datensätze|einträge|micrograms|µg|ug|mg|mmol|fl)(?![A-Za-zÀ-ÿ])|\s*→\s*\d+\s*(?:bits?|bytes?)(?![A-Za-z])|:\d|\|)/i;
567
+ const PID_BEFORE = /[A-Za-z0-9_\-]\[$/;
568
+ /** A standard's name as the letters of a lettered postcode: "RFC 5737". Not
569
+ * BS or EN: those are Bristol and Enfield. */
570
+ const STANDARD_LETTERS = /^(?:RFC|ISO|IEC|DIN)\s?\d/;
571
+ /** A year closing a range: "1998–2011". En and em dashes only. */
572
+ const YEAR_RANGE_BEFORE = /(?:19|20)\d\d\s?[–—]\s?$/;
573
+ const YEAR_RANGE_AFTER = /^\s?[–—]\s?(?:19|20)\d\d(?!\d)/;
509
574
  /** A thousands group of an amount: the "750" of "£28,750,000". */
510
575
  const GROUP_BEFORE = /\d[,.]$/;
511
576
  const GROUP_AFTER = /^[,.]\d{3}(?!\d)/;
@@ -519,10 +584,90 @@ function suppressPostalAfterDisqualifyingWord(text, match) {
519
584
  return true;
520
585
  if (GROUP_BEFORE.test(before) && GROUP_AFTER.test(text.slice(match.end, match.end + 5)))
521
586
  return true;
587
+ const after = text.slice(match.end, match.end + 24);
588
+ if (TECH_WORD_BEFORE.test(before) || TECH_SHAPE_BEFORE.test(before) || POSTAL_UNIT_AFTER.test(after)
589
+ || STANDARD_LETTERS.test(match.text))
590
+ return true;
591
+ if (PID_BEFORE.test(before) && after.startsWith("]"))
592
+ return true;
593
+ if (DATE_YEAR.test(match.text.trim()) && (YEAR_RANGE_BEFORE.test(before) || YEAR_RANGE_AFTER.test(after)))
594
+ return true;
522
595
  return RECENT_YEAR.test(match.text.trim())
523
596
  && (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
524
597
  && !TOWN_AFTER.test(text.slice(match.end, match.end + 3));
525
598
  }
599
+ /** A group with a leading zero: "2.00.079.01" is a version, not an address. */
600
+ const IP_LEADING_ZERO = /(?:^|\.)0\d/;
601
+ const CIDR_PREFIX_AFTER = /^\/(\d{1,2})(?!\d)/;
602
+ /** RFC 5737's documentation ranges: reserved, never routed, used in examples. */
603
+ const DOCUMENTATION_NETS = ["192.0.2.", "198.51.100.", "203.0.113."];
604
+ /**
605
+ * An IPv4 shape that names no host (rules-engine#126): a network address
606
+ * ("10.40.0.0/16"), a documentation address, or a version with leading zeros.
607
+ * An interface address with its prefix ("192.168.1.23/24") stays.
608
+ */
609
+ function suppressIpNotAHost(text, match) {
610
+ if (match.patternDef.entityType !== types_js_1.EntityType.IP_ADDRESS)
611
+ return false;
612
+ const ip = match.text;
613
+ if (IP_LEADING_ZERO.test(ip) || DOCUMENTATION_NETS.some((n) => ip.startsWith(n)))
614
+ return true;
615
+ // Part of a longer dotted run: "urn:oid:1.2.250.1.71.4.2.1" is an OID.
616
+ if (/\d\.$/.test(text.slice(Math.max(0, match.start - 2), match.start))
617
+ || /^\.\d/.test(text.slice(match.end, match.end + 2)))
618
+ return true;
619
+ const prefix = CIDR_PREFIX_AFTER.exec(text.slice(match.end, match.end + 4));
620
+ if (prefix !== null && Number(prefix[1]) < 32) {
621
+ const value = ip.split(".").reduce((acc, o) => acc * 256 + Number(o), 0);
622
+ return value % 2 ** (32 - Number(prefix[1])) === 0;
623
+ }
624
+ return false;
625
+ }
626
+ /**
627
+ * The specimen of a form: "das Muster „C01X00T47“". Only "Muster": "z. B." and
628
+ * "e.g." introduce real values in prose as often as samples (rules-engine#126).
629
+ */
630
+ const EXAMPLE_BEFORE = /(?<![A-Za-z])muster\s*[:,]?\s*[„"«']?$/i;
631
+ /** A dummy value in placeholder glyphs ("XX0000000", "999 999 9999") or MAC
632
+ * groups of one repeated letter. Not "111222333", the standard test BSN. */
633
+ const DUMMY_VALUE = /^(?=(?:[\s:\-./]*[X09]){5})(?:[X\s:\-./]*[X][X0\s:\-./]*|[X9\s:\-./]*X[X9\s:\-./]*|[0\s:\-./]+|[9\s:\-./]+)$|^(?:([a-fA-F])\1[:\-]){2,}([a-fA-F])\2$/;
634
+ /** Algorithm names in a key or signature line: "ED25519", "SHA256". */
635
+ const ALGORITHM = /^(?:ED25519|ED448|SHA-?(?:1|224|256|384|512)|MD5|AES-?\d{3}|RSA-?\d{3,4}|[HRE]S(?:256|384|512))$/i;
636
+ /** A date exactly, which no national number is written as. */
637
+ const EXACT_DOTTED_DATE = /^(?:0[1-9]|[12]\d|3[01])\.(?:0[1-9]|1[0-2])\.(?:19|20)\d\d$/;
638
+ /** The identifier types an example marker or a dummy value rules out. */
639
+ const EXAMPLE_TYPES = new Set([
640
+ types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.PASSPORT, types_js_1.EntityType.LICENSE_PLATE,
641
+ types_js_1.EntityType.HEALTH_INSURANCE, types_js_1.EntityType.MAC_ADDRESS,
642
+ types_js_1.EntityType.DRIVERS_LICENSE, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID,
643
+ ]);
644
+ /** A placeholder in the shape of a value, for any type (#126). */
645
+ function suppressDummyValue(_text, match) {
646
+ return DUMMY_VALUE.test(match.text.trim());
647
+ }
648
+ /** An identifier given as a form's specimen, not anyone's (#126). */
649
+ function suppressExampleValue(text, match) {
650
+ if (!EXAMPLE_TYPES.has(match.patternDef.entityType))
651
+ return false;
652
+ return EXAMPLE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
653
+ }
654
+ /**
655
+ * Shapes that fit an identifier pattern and are something else (#126): a
656
+ * passport without a digit, an algorithm or a law citation read as a plate,
657
+ * a date read as a dotted national number.
658
+ */
659
+ function suppressNotAnIdentifier(text, match) {
660
+ const type = match.patternDef.entityType;
661
+ const value = match.text.trim();
662
+ if (type === types_js_1.EntityType.PASSPORT)
663
+ return !/\d/.test(value);
664
+ if (type === types_js_1.EntityType.LICENSE_PLATE) {
665
+ return ALGORITHM.test(value) || /§\s*$/.test(text.slice(Math.max(0, match.start - 4), match.start));
666
+ }
667
+ if (type === types_js_1.EntityType.NATIONAL_ID)
668
+ return EXACT_DOTTED_DATE.test(value);
669
+ return false;
670
+ }
526
671
  function suppressPhoneServiceNumber(_text, match) {
527
672
  if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
528
673
  return false;
@@ -634,7 +779,11 @@ function suppressDePlateUnknownDistrict(text, match) {
634
779
  const parts = match.text.trim().split(/[\s\-]+/);
635
780
  if (parts.length === 0 || !parts[0])
636
781
  return false;
637
- if (deDistricts_js_1.DE_DISTRICT_CODES.has(parts[0].toUpperCase()))
782
+ // A German plate number never starts with 0: without a plate cue,
783
+ // "GR-NB-0412" is an asset tag and "HR-IT-03" a form (rules-engine#126).
784
+ const digits = /\d+/.exec(match.text);
785
+ const leadingZero = digits !== null && digits[0].startsWith("0");
786
+ if (deDistricts_js_1.DE_DISTRICT_CODES.has(parts[0].toUpperCase()) && !leadingZero)
638
787
  return false;
639
788
  const [before, after] = getContext(text, match.start, match.end);
640
789
  if (PLATE_CUE_NEAR.test(before + after))
@@ -1076,15 +1225,22 @@ function suppressRequiresContext(text, match) {
1076
1225
  return false;
1077
1226
  // A value filling an entire field of a delimited row is introduced by its
1078
1227
  // column, not by a word, so that is context — but only for narrow shapes.
1228
+ // A date's column decides when the table names it: under "début" or
1229
+ // "last_day" a date is not a birth date (rules-engine#126).
1079
1230
  if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
1080
- fillsADelimitedField(text, match.start, match.end))
1231
+ fillsADelimitedField(text, match.start, match.end) &&
1232
+ !(DATE_LABEL_TYPES.has(match.patternDef.entityType) && columnHeader(text, match.start) !== null))
1081
1233
  return false;
1082
1234
  if (DATE_LABEL_TYPES.has(match.patternDef.entityType)) {
1083
1235
  return !labelsThisDate(text, match.start, match.end, match.patternDef.contextKeywords, match.patternDef.entityType);
1084
1236
  }
1085
1237
  const [before, after] = getContext(text, match.start, match.end);
1086
1238
  const context = (before + " " + after).toLowerCase();
1087
- return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
1239
+ if (match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase())))
1240
+ return false;
1241
+ // The column names it: a passport under "Passport" rows below the header (#130).
1242
+ const header = columnHeader(text, match.start);
1243
+ return header === null || !match.patternDef.contextKeywords.some(kw => keywordAt(header, kw, false) >= 0);
1088
1244
  }
1089
1245
  /** Date types decided by a label: DOB and DATE_OF_DEATH share one date shape. */
1090
1246
  const DATE_LABEL_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
@@ -1120,10 +1276,16 @@ function nothingButAListBetween(gap) {
1120
1276
  */
1121
1277
  function keywordAt(haystack, keyword, last) {
1122
1278
  const kw = keyword.toLowerCase();
1279
+ const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
1123
1280
  if (keyword !== keyword.toUpperCase() || keyword === kw) {
1124
- return last ? haystack.lastIndexOf(kw) : haystack.indexOf(kw);
1281
+ // It must start a word, though it may run on into a longer one: "née le"
1282
+ // in "signée le" and "born" in "Newborn" are other words (rules-engine#126).
1283
+ let i = last ? haystack.lastIndexOf(kw) : haystack.indexOf(kw);
1284
+ while (i > 0 && isLetter(haystack[i - 1])) {
1285
+ i = last ? (i === 0 ? -1 : haystack.lastIndexOf(kw, i - 1)) : haystack.indexOf(kw, i + 1);
1286
+ }
1287
+ return i;
1125
1288
  }
1126
- const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
1127
1289
  const hits = [];
1128
1290
  for (let i = haystack.indexOf(kw); i !== -1; i = haystack.indexOf(kw, i + 1)) {
1129
1291
  let next = i + kw.length;
@@ -1185,29 +1347,44 @@ function columnHeader(text, start) {
1185
1347
  const lineStart = text.lastIndexOf("\n", start - 1) + 1;
1186
1348
  const lineEndAt = text.indexOf("\n", start);
1187
1349
  const line = text.slice(lineStart, lineEndAt >= 0 ? lineEndAt : text.length);
1188
- // The three lines above, by lastIndexOf: splitting the whole prefix was
1189
- // quadratic in the document, once per date candidate.
1190
- let aboveStart = lineStart - 1;
1191
- for (let k = 0; k < 3 && aboveStart >= 0; k++) {
1192
- aboveStart = aboveStart === 0 ? -1 : text.lastIndexOf("\n", aboveStart - 1);
1193
- }
1194
- const above = lineStart ? text.slice(aboveStart + 1, Math.max(0, lineStart - 1)).split("\n") : [];
1195
1350
  for (const sep of COLUMN_SEPARATORS) {
1196
1351
  if (!line.includes(sep))
1197
1352
  continue;
1198
1353
  const cells = line.split(sep);
1199
1354
  const column = text.slice(lineStart, start).split(sep).length - 1;
1200
- for (const header of [...above].reverse()) {
1201
- // A "label : value" field with a comma in its value is not a header (#111).
1202
- if (header.includes(":"))
1355
+ // Lines above, one lastIndexOf at a time: one of the three lines above, or
1356
+ // further up through rows of the same table (rules-engine#126).
1357
+ let end = lineStart - 1;
1358
+ let seen = 0;
1359
+ while (end >= 0 && seen < HEADER_REACH) {
1360
+ const begin = end === 0 ? 0 : text.lastIndexOf("\n", end - 1) + 1;
1361
+ const header = text.slice(begin, end);
1362
+ seen++;
1363
+ // A rule line ("|---|---|", "----+----") is neither header nor row (#130).
1364
+ if (RULE_LINE.test(header)) {
1365
+ end = begin - 1;
1203
1366
  continue;
1367
+ }
1204
1368
  const heads = header.split(sep);
1205
- if (heads.length === cells.length && !ANY_DATE.test(header))
1369
+ const same = heads.length === cells.length;
1370
+ // A "label : value" field with a comma in its value is not a header
1371
+ // (#111); a commented header ("# Spalten: Name;von;bis") is (#126).
1372
+ // Nor is a data row without a date: a header has no cell with four digits or more (#130).
1373
+ if (same && (!header.includes(":") || header.trimStart().startsWith("#")) && !ANY_DATE.test(header)
1374
+ && !heads.some((h) => (h.match(/\d/g) ?? []).length >= 4)) {
1206
1375
  return heads[column].toLowerCase();
1376
+ }
1377
+ if (seen >= 3 && !same)
1378
+ break; // past the table this value sits in
1379
+ end = begin - 1;
1207
1380
  }
1208
1381
  }
1209
1382
  return null;
1210
1383
  }
1384
+ /** How many lines up a table's header is looked for. */
1385
+ const HEADER_REACH = 200;
1386
+ /** A table's rule line: only dashes, equals signs, plus signs, colons, pipes. */
1387
+ const RULE_LINE = /^[\s\-=+:|]*-[\s\-=+:|]*$/;
1211
1388
  /** A dotted MAC shape glued to more of a reference is part of it: the tail of
1212
1389
  * "WSO-II.6151.4471.2025" (rules-engine#105). Dotted form only. */
1213
1390
  function suppressMacInReference(text, match) {
@@ -1242,12 +1419,18 @@ const TYPE_SUPPRESSORS = {
1242
1419
  [types_js_1.EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
1243
1420
  [types_js_1.EntityType.MAC_ADDRESS]: [suppressMacInReference],
1244
1421
  [types_js_1.EntityType.CREDIT_CARD]: [suppressCardInsideDigitRun],
1422
+ [types_js_1.EntityType.IP_ADDRESS]: [suppressIpNotAHost],
1245
1423
  };
1424
+ for (const type of EXAMPLE_TYPES) {
1425
+ TYPE_SUPPRESSORS[type] = [...(TYPE_SUPPRESSORS[type] ?? []), suppressExampleValue, suppressNotAnIdentifier];
1426
+ }
1246
1427
  function shouldSuppress(text, match, scratch) {
1247
1428
  if (suppressSequential(text, match))
1248
1429
  return true;
1249
1430
  if (suppressRedactionPlaceholder(text, match))
1250
1431
  return true;
1432
+ if (suppressDummyValue(text, match))
1433
+ return true;
1251
1434
  const typeSups = TYPE_SUPPRESSORS[match.patternDef.entityType];
1252
1435
  if (typeSups) {
1253
1436
  for (const s of typeSups) {