euredact 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +455 -44
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +124 -14
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +83 -13
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +479 -26
- package/dist/cjs/rules/validators.js +50 -0
- package/dist/cjs/sdk.js +108 -14
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +83 -1
- package/dist/esm/cloud/client.js +119 -22
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +83 -13
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +478 -25
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +47 -0
- package/dist/esm/sdk.d.ts +77 -4
- package/dist/esm/sdk.js +108 -17
- package/dist/esm/types.d.ts +64 -6
- package/dist/esm/types.js +4 -1
- package/package.json +13 -4
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EntityType } from "../types.js";
|
|
2
|
-
import { isRegisteredBic } from "./bicRegistry.js";
|
|
2
|
+
import { isKnownInstitution, isRegisteredBic } from "./bicRegistry.js";
|
|
3
3
|
import { DE_DISTRICT_CODES } from "./deDistricts.js";
|
|
4
4
|
const CONTEXT_CHARS = 150;
|
|
5
5
|
function getContext(text, start, end) {
|
|
@@ -21,10 +21,11 @@ const CURRENCY_WORD = "EUR|USD|GBP|CHF|ISK|SEK|NOK|DKK|PLN|CZK|HUF|RON|BGN|HRK|"
|
|
|
21
21
|
const CURRENCY_AFTER = new RegExp(`^\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
22
22
|
const CURRENCY_COMMA_AFTER = new RegExp(`^[.,]\\d{1,2}\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
23
23
|
const CURRENCY_BEFORE = new RegExp(`(?:(?:${CURRENCY_SYMBOL})|\\b(?:${CURRENCY_WORD}))\\s*$`, "i");
|
|
24
|
-
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)
|
|
24
|
+
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)(?:\s+\w+)?\s*:?\s*$/i; // second word separate: not "Rent"+"enversicherungsnummer" (rules-engine#111)
|
|
25
25
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
26
26
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
27
|
-
|
|
27
|
+
// Horizontal space only: a unit is on the number's own line (rules-engine#51).
|
|
28
|
+
const UNIT_AFTER = /^[ \t]*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
28
29
|
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
29
30
|
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|ticket|incident\s*(?:report|nr|no)?|case\s*(?:nr|no|id)?|zaaknummer|meldingsnummer|Vorgangsnummer|Vorgang|Störungsmeldung|saksnummer|ärendenummer|sagsnummer|asianumero|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.#]?\s*$/i;
|
|
30
31
|
// "#" before a number marks a reference in every language in scope — a ticket,
|
|
@@ -32,9 +33,19 @@ const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|f
|
|
|
32
33
|
const HASH_BEFORE = /#\s*$/;
|
|
33
34
|
// A short uppercase tag hyphenated to the number: "IR-43433", "INC-2024".
|
|
34
35
|
const REF_PREFIX_BEFORE = /(?:^|[\s([])[A-Z]{2,5}-$/;
|
|
36
|
+
// The cross-border address form, "CH-8004 Zürich": a country code hyphenated to
|
|
37
|
+
// a postal code, told apart from a reference tag by position -- it opens an
|
|
38
|
+
// address line and a place name follows (rules-engine#58).
|
|
39
|
+
const COUNTRY_PREFIX_OPENS_LINE = /(?:^|\n|,)[ \t]*([A-Z]{2})-$/;
|
|
40
|
+
const POSTAL_PREFIX_COUNTRIES = new Set([
|
|
41
|
+
"AT", "BE", "BG", "CH", "CY", "CZ", "DE", "DK", "EE", "EL", "ES", "FI",
|
|
42
|
+
"FR", "GB", "GR", "HR", "HU", "IE", "IS", "IT", "LI", "LT", "LU", "LV",
|
|
43
|
+
"MT", "NL", "NO", "PL", "PT", "RO", "SE", "SI", "SK", "UK",
|
|
44
|
+
]);
|
|
35
45
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
36
|
-
|
|
37
|
-
const
|
|
46
|
+
// An operator on the same line only: a bullet or "---" on the next line is not a minus sign (rules-engine#91).
|
|
47
|
+
const MATH_BEFORE = /[=+\-×÷*/][^\S\r\n]*$/;
|
|
48
|
+
const MATH_AFTER = /^[^\S\r\n]*[=+\-×÷*/]/;
|
|
38
49
|
// "A-1010", "B-2000", "L-1234", "CH-8000", "D-10115": a country prefix on a
|
|
39
50
|
// postal code, not a minus sign.
|
|
40
51
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
@@ -60,10 +71,11 @@ const NUMERIC_TYPES = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityT
|
|
|
60
71
|
// Month names in the languages this engine covers. Long, but it is data: the
|
|
61
72
|
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
62
73
|
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
63
|
-
const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
74
|
+
export const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
64
75
|
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
65
76
|
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
66
77
|
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
78
|
+
"|janv|févr|fevr|avr|juil|déc" +
|
|
67
79
|
"|januari|februari|maart|mei|augustus|mrt" +
|
|
68
80
|
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
69
81
|
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
@@ -187,6 +199,27 @@ function suppressSecretOverStructured(text, match) {
|
|
|
187
199
|
* natural word. A random all-letter token would be mixed case and is left alone
|
|
188
200
|
* — losing a real secret is the expensive direction.
|
|
189
201
|
*/
|
|
202
|
+
/**
|
|
203
|
+
* Values an assigned-secret rule finds after a "=" or ":" that are names, not
|
|
204
|
+
* secrets (rules-engine#126): `${VAR}` references, identifiers, hyphenated
|
|
205
|
+
* compounds, image tags, product versions, code calls. Not paths, host names
|
|
206
|
+
* or reference numbers: those carry personal data ("mac-mhartmann88").
|
|
207
|
+
* Kept in sync with the Python SDK's _SECRET_NAME_SHAPES.
|
|
208
|
+
*/
|
|
209
|
+
const SECRET_NAME_SHAPES = new RegExp([
|
|
210
|
+
String.raw `^\$\{[A-Za-z_][A-Za-z0-9_]*(?::?[-=?+][^}]*)?\}?$`,
|
|
211
|
+
String.raw `^[a-z]{2,}(?:_[a-z]{2,})+$`, String.raw `^[A-Z]{2,}(?:_[A-Z]{2,})+$`,
|
|
212
|
+
String.raw `^(?:[A-Z]+[a-z]{2,}){3,}(?:V?\d{1,2})?$`,
|
|
213
|
+
String.raw `^[a-z][a-z0-9._/-]*:(?:\d+(?:\.\d+)*(?:-[a-z0-9]+)*|latest|stable|alpine|slim)$`,
|
|
214
|
+
String.raw `^[A-Za-z][\w.-]*\/\d+(?:\.\d+)+$`,
|
|
215
|
+
String.raw `^[\w.]+\(["'\x60]`,
|
|
216
|
+
].join("|"));
|
|
217
|
+
/** A hyphenated compound of words and acronyms; two capitalised words alone
|
|
218
|
+
* ("Ann-Sophie", "Caron-Cordier") are names and are left alone. */
|
|
219
|
+
const COMPOUND_PART = String.raw `(?:[A-ZÀ-Þ]{2,6}\d{0,3}|[A-Za-zÀ-ÿ][a-zß-ÿ]{2,})`;
|
|
220
|
+
const COMPOUND = new RegExp(String.raw `^` + COMPOUND_PART + String.raw `(?:-` + COMPOUND_PART + String.raw `)+$`);
|
|
221
|
+
const NAME_PAIR = /^[A-ZÀ-Þ][a-zß-ÿ]{2,}-[A-ZÀ-Þ][a-zß-ÿ]{2,}$/;
|
|
222
|
+
const CONNECTION_WITH_REFERENCE = /^[a-z]+:\/\/[^\s:@]+:\$\{[^}]*\}@/;
|
|
190
223
|
function suppressSecretNotASecret(_text, match) {
|
|
191
224
|
if (match.patternDef.entityType !== EntityType.SECRET)
|
|
192
225
|
return false;
|
|
@@ -201,6 +234,10 @@ function suppressSecretNotASecret(_text, match) {
|
|
|
201
234
|
return true;
|
|
202
235
|
if (HYPHENATED_WORDS.test(bare))
|
|
203
236
|
return true;
|
|
237
|
+
if (SECRET_NAME_SHAPES.test(bare) || CONNECTION_WITH_REFERENCE.test(bare))
|
|
238
|
+
return true;
|
|
239
|
+
if (COMPOUND.test(bare) && !NAME_PAIR.test(bare))
|
|
240
|
+
return true;
|
|
204
241
|
const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
|
|
205
242
|
const oneWord = bare === bare.toLowerCase() ||
|
|
206
243
|
(bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
|
|
@@ -285,7 +322,19 @@ function suppressReference(text, match) {
|
|
|
285
322
|
if (REFERENCE_BEFORE.test(before))
|
|
286
323
|
return true;
|
|
287
324
|
const adjacent = text.slice(Math.max(0, match.start - 8), match.start);
|
|
288
|
-
|
|
325
|
+
if (HASH_BEFORE.test(adjacent))
|
|
326
|
+
return true;
|
|
327
|
+
if (!REF_PREFIX_BEFORE.test(adjacent))
|
|
328
|
+
return false;
|
|
329
|
+
return !(match.patternDef.entityType === EntityType.POSTAL_CODE &&
|
|
330
|
+
isCountryPrefixedAddress(text, match.start, match.end));
|
|
331
|
+
}
|
|
332
|
+
/** "CH-8004 Zürich" on an address line, not the reference "IT-20431". */
|
|
333
|
+
function isCountryPrefixedAddress(text, start, end) {
|
|
334
|
+
const found = COUNTRY_PREFIX_OPENS_LINE.exec(text.slice(Math.max(0, start - 12), start));
|
|
335
|
+
return (found !== null &&
|
|
336
|
+
POSTAL_PREFIX_COUNTRIES.has(found[1]) &&
|
|
337
|
+
CITY_AFTER.test(text.slice(end, end + 24)));
|
|
289
338
|
}
|
|
290
339
|
function suppressLegal(text, match) {
|
|
291
340
|
const applicable = LEGAL_TYPES;
|
|
@@ -330,7 +379,20 @@ function suppressSequential(_text, match) {
|
|
|
330
379
|
// the "]". The opening bracket is still required.
|
|
331
380
|
const PLACEHOLDER = /^(?:\[([A-Z][A-Z0-9_]*)\]?|([A-Z][A-Z0-9_]*?)_(?:\d+|[A-HJ-NP-Z2-9]{4}))$/;
|
|
332
381
|
const KNOWN_TYPE_NAMES = new Set(Object.values(EntityType));
|
|
333
|
-
function suppressRedactionPlaceholder(
|
|
382
|
+
function suppressRedactionPlaceholder(text, match) {
|
|
383
|
+
// The name between the brackets: an assigned value now starts after an
|
|
384
|
+
// opening bracket, so "credentials: [POSTAL_CODE]" offers "POSTAL_CODE" alone
|
|
385
|
+
// (rules-engine#33, #126).
|
|
386
|
+
if (text[match.start - 1] === "[" && text[match.end] === "]" && KNOWN_TYPE_NAMES.has(match.text))
|
|
387
|
+
return true;
|
|
388
|
+
// A placeholder inside the span: on text masked once, a value running across
|
|
389
|
+
// "[IP_ADDRESS]" is two things joined (rules-engine#126).
|
|
390
|
+
// One character either side: a span may start just inside the bracket.
|
|
391
|
+
const around = text.slice(Math.max(0, match.start - 1), match.end + 1);
|
|
392
|
+
for (const m of around.matchAll(/\[([A-Z][A-Z0-9_]*)\]/g)) {
|
|
393
|
+
if (KNOWN_TYPE_NAMES.has(m[1]))
|
|
394
|
+
return true;
|
|
395
|
+
}
|
|
334
396
|
const found = PLACEHOLDER.exec(match.text);
|
|
335
397
|
if (found === null)
|
|
336
398
|
return false;
|
|
@@ -338,6 +400,8 @@ function suppressRedactionPlaceholder(_text, match) {
|
|
|
338
400
|
return KNOWN_TYPE_NAMES.has(name);
|
|
339
401
|
}
|
|
340
402
|
const RECENT_YEAR = /^(?:19[5-9]\d|20[0-3]\d)$/;
|
|
403
|
+
// Any year a written date can carry: the date-construction test applies to all of them (rules-engine#111).
|
|
404
|
+
const DATE_YEAR = /^(?:1[89]\d\d|20\d\d)$/;
|
|
341
405
|
const POSTAL_CONTEXT_NEAR = /(?:postcode|postal|code\s*postal|PLZ|Postleitzahl|postnummer|postinumero|póstnúmer|zip|straat|straße|strasse|rue\s|via\s|calle\s|rua\s|ulica|utca|street|avenue|laan\s|weg\s|plein|adres|adresse|address|woonplaats|wonende|woonachtig|gevestigd|domicili|demeurant|résidant|residant|bosatt|bopæl|wohnhaft|ansässig|stad\b|ville\b|city\b|Stadt|città|ciudad|cidade|miasto|město|város)/i;
|
|
342
406
|
const DATE_KEYWORD_NEAR = /(?:født|fødselsdato|fødsel|Fødselsdato|född|födelsedatum|födelsedag|syntynyt|syntymäaika|fæddur|fæðingardagur|geboren|geboortedatum|Geburtsdatum|nascido|nacido|data di nascita|nato il|nata il|Tiltr[æa]delsesdato|Tiltredelsesdato|datum|date\b|Datum|jaar|year|année|Jahr|since|sinds|depuis|seit|\d{2}\.\d{2}\.|(?:januar|februar|marts|april|maj|juni|juli|august|september|oktober|november|december|januari|februari|mars|mei|juin|juillet|août))/i;
|
|
343
407
|
// A sentence boundary: a full stop, question or exclamation mark followed by
|
|
@@ -367,14 +431,27 @@ function sentenceAround(text, start, end, reach = 160) {
|
|
|
367
431
|
const POSTAL_LABEL_TOUCHING = new RegExp(`(?:${POSTAL_CONTEXT_NEAR.source})[\\s:.\\-]{0,4}$`, "i");
|
|
368
432
|
// A capitalised place name after the code: "2000 Antwerpen". A following label
|
|
369
433
|
// ("1970 Fødselsnummer:") is excluded by the colon.
|
|
370
|
-
|
|
434
|
+
// `\p{L}\p{N}_` rather than `\w`: JS `\w` is ASCII-only, so "8004 Zürich" failed
|
|
435
|
+
// at the ü here while Python's Unicode `\w` accepted it (rules-engine#58).
|
|
436
|
+
const CITY_AFTER = /^[ \t](?![\p{L}\p{N}_]+\s*:)[A-ZÀ-ÞŁŠŽ][\p{L}\p{N}_\-']{2,}/u;
|
|
371
437
|
// Every position is local to the value or inside its own sentence. Proximity to
|
|
372
438
|
// an address *word* anywhere in a 300-character window is what the old rescue
|
|
373
439
|
// asked, and that is a neighbourhood rather than a position (rules-engine#32).
|
|
440
|
+
/**
|
|
441
|
+
* The comma of an address line: the word before it is a place name or a number.
|
|
442
|
+
* "myocardial infarction, 2016" is a year in a list (rules-engine#111).
|
|
443
|
+
*/
|
|
444
|
+
const ADDRESS_COMMA = /(?<!\S)(?:[A-ZÀ-Þ\d][^\s,]*|[^\s,]*\]),[ \t]*$/;
|
|
445
|
+
/**
|
|
446
|
+
* A redaction placeholder: "[POSTAL_CODE]" holds the words the address test
|
|
447
|
+
* looks for, so on masked text it made every year in its sentence a postcode
|
|
448
|
+
* (rules-engine#111). A masked value is no address word.
|
|
449
|
+
*/
|
|
450
|
+
const MASKED_VALUE = /\[[A-Z][A-Z_]*\]/g;
|
|
374
451
|
function yearSitsInAddressStructure(text, start, end) {
|
|
375
452
|
const before = text.slice(Math.max(0, start - 28), start);
|
|
376
453
|
// "Amsterdam, 2026" -- the comma of an address line.
|
|
377
|
-
if (
|
|
454
|
+
if (ADDRESS_COMMA.test(before))
|
|
378
455
|
return true;
|
|
379
456
|
// "PLZ: 2011" -- a postal label with nothing but punctuation between.
|
|
380
457
|
if (POSTAL_LABEL_TOUCHING.test(before))
|
|
@@ -386,13 +463,13 @@ function yearSitsInAddressStructure(text, start, end) {
|
|
|
386
463
|
// 12, rustige ligging in 2018, vlakbij openbaar vervoer." is an Antwerp
|
|
387
464
|
// postal code in prose; the false positives put the year and the address in
|
|
388
465
|
// different sentences.
|
|
389
|
-
return POSTAL_CONTEXT_NEAR.test(sentenceAround(text, start, end));
|
|
466
|
+
return POSTAL_CONTEXT_NEAR.test(sentenceAround(text, start, end).replace(MASKED_VALUE, " "));
|
|
390
467
|
}
|
|
391
468
|
function suppressYearAsPostal(text, match) {
|
|
392
469
|
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
393
470
|
return false;
|
|
394
471
|
const clean = match.text.trim();
|
|
395
|
-
if (!
|
|
472
|
+
if (!DATE_YEAR.test(clean))
|
|
396
473
|
return false;
|
|
397
474
|
// A date construction *touching* the candidate settles it, whatever the rest
|
|
398
475
|
// of the document says. Without this, the postal-context test below rescued
|
|
@@ -404,6 +481,8 @@ function suppressYearAsPostal(text, match) {
|
|
|
404
481
|
return true;
|
|
405
482
|
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
406
483
|
return true;
|
|
484
|
+
if (!RECENT_YEAR.test(clean))
|
|
485
|
+
return false;
|
|
407
486
|
const [before, after] = getContext(text, match.start, match.end);
|
|
408
487
|
const context = before + after;
|
|
409
488
|
// Keep as postal code only when the value sits in address *structure* --
|
|
@@ -411,10 +490,6 @@ function suppressYearAsPostal(text, match) {
|
|
|
411
490
|
// words was the defect rather than the keyword list.
|
|
412
491
|
if (yearSitsInAddressStructure(text, match.start, match.end))
|
|
413
492
|
return false;
|
|
414
|
-
// Keep if preceded by comma+space (address pattern: "Amsterdam, 2026")
|
|
415
|
-
const immediateBefore = text.slice(Math.max(0, match.start - 3), match.start);
|
|
416
|
-
if (/,\s*$/.test(immediateBefore))
|
|
417
|
-
return false;
|
|
418
493
|
// Suppress if date keyword nearby (birth date, employment date, etc.)
|
|
419
494
|
if (DATE_KEYWORD_NEAR.test(context))
|
|
420
495
|
return true;
|
|
@@ -457,26 +532,155 @@ const TOWN_AFTER = /^\s*[A-ZÀ-ÞĀ-Ž]/;
|
|
|
457
532
|
* between the marker and the number, so allow exactly one.
|
|
458
533
|
*/
|
|
459
534
|
const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|durchwahl|doorkiesnummer|tst|nebenstelle)\.?\s*(?:[a-zà-ÿ]{2,10}\s*)?[:.]?\s*$/i;
|
|
535
|
+
/**
|
|
536
|
+
* What else a bare digit run after these is (rules-engine#111): a head count
|
|
537
|
+
* ("Employees: 214"), a standard ("ISO/IEC 27001"), a product model with an
|
|
538
|
+
* inner capital ("Dell OptiPlex 7090"). Not the digits after "[PHONE]": the
|
|
539
|
+
* rest of a number an earlier pass cut short, safer masked as a postcode.
|
|
540
|
+
*/
|
|
541
|
+
const COUNT_LABEL_BEFORE = /(?<![A-Za-z0-9_])(?:employees|headcount|staff|mitarbeiter(?:innen)?|beschäftigte|werknemers|medewerkers|employés|salariés|effectifs?|dipendenti|empleados|pracowników|zaměstnanců)\s*[:=]?\s*$/i;
|
|
542
|
+
const NOT_POSTAL_BEFORE = /(?<![A-Za-z0-9_])(?:(?:ISO(?:\/IEC)?|IEC|EN|DIN(?:\s+EN)?|NEN(?:-EN)?|NF\s+EN|BS(?:\s+EN)?|PN-EN|ÖNORM)[\s\-]*|[A-Za-z]*[a-z][A-Z][A-Za-z]*\s+)$/;
|
|
543
|
+
/**
|
|
544
|
+
* What else a number is, by what touches it (rules-engine#126): a build,
|
|
545
|
+
* version, event, lease, policy or ticket number; a port; an invoice, reference
|
|
546
|
+
* or study-subject number; a process id; a JSON id or port field; an HTTP
|
|
547
|
+
* response size; the hour after a date; a lab reference range; or a quantity
|
|
548
|
+
* with its unit after it. Kept in sync with the Python SDK.
|
|
549
|
+
*/
|
|
550
|
+
const TECH_WORD_BEFORE = /(?<![A-Za-z0-9_])(?:build|version|policy\s+id|ticket(?:\s+[A-Za-z]{2,10})?|event|lease|pid|job|port|rfc|facture|invoice|r[ée]f[ée]rence|n[°º]|tn|sae|uczestnik|subject|narratief|participant)\.?\s*[:#=]?\s*["(«[]?\s*$/i;
|
|
551
|
+
const TECH_SHAPE_BEFORE = new RegExp("(?:" + [
|
|
552
|
+
String.raw `"[A-Za-z_]*(?:id|port)"\s*:\s*"?`, // "parent_id": 4400
|
|
553
|
+
String.raw `(?<![A-Za-z0-9])[A-Z_]*PORT\s*[:=]\s*"?`, // DB_PORT: "5432"
|
|
554
|
+
String.raw `(?:localhost|[a-z0-9\-]+\.[a-z0-9.\-]+):`, // localhost:3000
|
|
555
|
+
String.raw `(?<![\d.])\d{2,5}:`, // "8080:8080"
|
|
556
|
+
String.raw `HTTP\/\d(?:\.\d)?"?\s+\d{3}\s+`, // HTTP/1.1" 200 2048
|
|
557
|
+
String.raw `\d+\.\d+(?:\.\d+)+\s*\(`, // build 2.4.0 (4012)
|
|
558
|
+
String.raw `(?<!\d)\d{1,2}[./]\d{1,2}[./]`, // 03.10.2026 23:50
|
|
559
|
+
String.raw `\((?:N|norm(?:al)?|ref)\s*:?\s*`, // (N : 30-400)
|
|
560
|
+
String.raw `\|`,
|
|
561
|
+
].join("|") + ")$", "i");
|
|
562
|
+
const POSTAL_UNIT_AFTER = /^(?:\s?(?:ms|s|sec|secs|seconds?|secondes?|sekunden|sek|min|bits?|bytes?|kb|mb|gb|tb|kbps|mbps|px|%|records|rows|entries|datensätze|einträge|micrograms|µg|ug|mg|mmol|fl)(?![A-Za-zÀ-ÿ])|\s*→\s*\d+\s*(?:bits?|bytes?)(?![A-Za-z])|:\d|\|)/i;
|
|
563
|
+
const PID_BEFORE = /[A-Za-z0-9_\-]\[$/;
|
|
564
|
+
/** A standard's name as the letters of a lettered postcode: "RFC 5737". Not
|
|
565
|
+
* BS or EN: those are Bristol and Enfield. */
|
|
566
|
+
const STANDARD_LETTERS = /^(?:RFC|ISO|IEC|DIN)\s?\d/;
|
|
567
|
+
/** A year closing a range: "1998–2011". En and em dashes only. */
|
|
568
|
+
const YEAR_RANGE_BEFORE = /(?:19|20)\d\d\s?[–—]\s?$/;
|
|
569
|
+
const YEAR_RANGE_AFTER = /^\s?[–—]\s?(?:19|20)\d\d(?!\d)/;
|
|
570
|
+
/** A thousands group of an amount: the "750" of "£28,750,000". */
|
|
571
|
+
const GROUP_BEFORE = /\d[,.]$/;
|
|
572
|
+
const GROUP_AFTER = /^[,.]\d{3}(?!\d)/;
|
|
460
573
|
/** Suppress a postal code the word in front of it rules out. */
|
|
461
574
|
function suppressPostalAfterDisqualifyingWord(text, match) {
|
|
462
575
|
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
463
576
|
return false;
|
|
464
577
|
const before = text.slice(Math.max(0, match.start - 24), match.start);
|
|
465
|
-
if (EXTENSION_BEFORE.test(before))
|
|
578
|
+
if (EXTENSION_BEFORE.test(before) || COUNT_LABEL_BEFORE.test(before)
|
|
579
|
+
|| NOT_POSTAL_BEFORE.test(before))
|
|
580
|
+
return true;
|
|
581
|
+
if (GROUP_BEFORE.test(before) && GROUP_AFTER.test(text.slice(match.end, match.end + 5)))
|
|
582
|
+
return true;
|
|
583
|
+
const after = text.slice(match.end, match.end + 24);
|
|
584
|
+
if (TECH_WORD_BEFORE.test(before) || TECH_SHAPE_BEFORE.test(before) || POSTAL_UNIT_AFTER.test(after)
|
|
585
|
+
|| STANDARD_LETTERS.test(match.text))
|
|
586
|
+
return true;
|
|
587
|
+
if (PID_BEFORE.test(before) && after.startsWith("]"))
|
|
588
|
+
return true;
|
|
589
|
+
if (DATE_YEAR.test(match.text.trim()) && (YEAR_RANGE_BEFORE.test(before) || YEAR_RANGE_AFTER.test(after)))
|
|
466
590
|
return true;
|
|
467
591
|
return RECENT_YEAR.test(match.text.trim())
|
|
468
592
|
&& (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
|
|
469
593
|
&& !TOWN_AFTER.test(text.slice(match.end, match.end + 3));
|
|
470
594
|
}
|
|
595
|
+
/** A group with a leading zero: "2.00.079.01" is a version, not an address. */
|
|
596
|
+
const IP_LEADING_ZERO = /(?:^|\.)0\d/;
|
|
597
|
+
const CIDR_PREFIX_AFTER = /^\/(\d{1,2})(?!\d)/;
|
|
598
|
+
/** RFC 5737's documentation ranges: reserved, never routed, used in examples. */
|
|
599
|
+
const DOCUMENTATION_NETS = ["192.0.2.", "198.51.100.", "203.0.113."];
|
|
600
|
+
/**
|
|
601
|
+
* An IPv4 shape that names no host (rules-engine#126): a network address
|
|
602
|
+
* ("10.40.0.0/16"), a documentation address, or a version with leading zeros.
|
|
603
|
+
* An interface address with its prefix ("192.168.1.23/24") stays.
|
|
604
|
+
*/
|
|
605
|
+
function suppressIpNotAHost(text, match) {
|
|
606
|
+
if (match.patternDef.entityType !== EntityType.IP_ADDRESS)
|
|
607
|
+
return false;
|
|
608
|
+
const ip = match.text;
|
|
609
|
+
if (IP_LEADING_ZERO.test(ip) || DOCUMENTATION_NETS.some((n) => ip.startsWith(n)))
|
|
610
|
+
return true;
|
|
611
|
+
// Part of a longer dotted run: "urn:oid:1.2.250.1.71.4.2.1" is an OID.
|
|
612
|
+
if (/\d\.$/.test(text.slice(Math.max(0, match.start - 2), match.start))
|
|
613
|
+
|| /^\.\d/.test(text.slice(match.end, match.end + 2)))
|
|
614
|
+
return true;
|
|
615
|
+
const prefix = CIDR_PREFIX_AFTER.exec(text.slice(match.end, match.end + 4));
|
|
616
|
+
if (prefix !== null && Number(prefix[1]) < 32) {
|
|
617
|
+
const value = ip.split(".").reduce((acc, o) => acc * 256 + Number(o), 0);
|
|
618
|
+
return value % 2 ** (32 - Number(prefix[1])) === 0;
|
|
619
|
+
}
|
|
620
|
+
return false;
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* The specimen of a form: "das Muster „C01X00T47“". Only "Muster": "z. B." and
|
|
624
|
+
* "e.g." introduce real values in prose as often as samples (rules-engine#126).
|
|
625
|
+
*/
|
|
626
|
+
const EXAMPLE_BEFORE = /(?<![A-Za-z])muster\s*[:,]?\s*[„"«']?$/i;
|
|
627
|
+
/** A dummy value in placeholder glyphs ("XX0000000", "999 999 9999") or MAC
|
|
628
|
+
* groups of one repeated letter. Not "111222333", the standard test BSN. */
|
|
629
|
+
const DUMMY_VALUE = /^(?=(?:[\s:\-./]*[X09]){5})(?:[X\s:\-./]*[X][X0\s:\-./]*|[X9\s:\-./]*X[X9\s:\-./]*|[0\s:\-./]+|[9\s:\-./]+)$|^(?:([a-fA-F])\1[:\-]){2,}([a-fA-F])\2$/;
|
|
630
|
+
/** Algorithm names in a key or signature line: "ED25519", "SHA256". */
|
|
631
|
+
const ALGORITHM = /^(?:ED25519|ED448|SHA-?(?:1|224|256|384|512)|MD5|AES-?\d{3}|RSA-?\d{3,4}|[HRE]S(?:256|384|512))$/i;
|
|
632
|
+
/** A date exactly, which no national number is written as. */
|
|
633
|
+
const EXACT_DOTTED_DATE = /^(?:0[1-9]|[12]\d|3[01])\.(?:0[1-9]|1[0-2])\.(?:19|20)\d\d$/;
|
|
634
|
+
/** The identifier types an example marker or a dummy value rules out. */
|
|
635
|
+
const EXAMPLE_TYPES = new Set([
|
|
636
|
+
EntityType.NATIONAL_ID, EntityType.PASSPORT, EntityType.LICENSE_PLATE,
|
|
637
|
+
EntityType.HEALTH_INSURANCE, EntityType.MAC_ADDRESS,
|
|
638
|
+
EntityType.DRIVERS_LICENSE, EntityType.SSN, EntityType.TAX_ID,
|
|
639
|
+
]);
|
|
640
|
+
/** A placeholder in the shape of a value, for any type (#126). */
|
|
641
|
+
function suppressDummyValue(_text, match) {
|
|
642
|
+
return DUMMY_VALUE.test(match.text.trim());
|
|
643
|
+
}
|
|
644
|
+
/** An identifier given as a form's specimen, not anyone's (#126). */
|
|
645
|
+
function suppressExampleValue(text, match) {
|
|
646
|
+
if (!EXAMPLE_TYPES.has(match.patternDef.entityType))
|
|
647
|
+
return false;
|
|
648
|
+
return EXAMPLE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
|
|
649
|
+
}
|
|
650
|
+
/**
|
|
651
|
+
* Shapes that fit an identifier pattern and are something else (#126): a
|
|
652
|
+
* passport without a digit, an algorithm or a law citation read as a plate,
|
|
653
|
+
* a date read as a dotted national number.
|
|
654
|
+
*/
|
|
655
|
+
function suppressNotAnIdentifier(text, match) {
|
|
656
|
+
const type = match.patternDef.entityType;
|
|
657
|
+
const value = match.text.trim();
|
|
658
|
+
if (type === EntityType.PASSPORT)
|
|
659
|
+
return !/\d/.test(value);
|
|
660
|
+
if (type === EntityType.LICENSE_PLATE) {
|
|
661
|
+
return ALGORITHM.test(value) || /§\s*$/.test(text.slice(Math.max(0, match.start - 4), match.start));
|
|
662
|
+
}
|
|
663
|
+
if (type === EntityType.NATIONAL_ID)
|
|
664
|
+
return EXACT_DOTTED_DATE.test(value);
|
|
665
|
+
return false;
|
|
666
|
+
}
|
|
471
667
|
function suppressPhoneServiceNumber(_text, match) {
|
|
472
668
|
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
473
669
|
return false;
|
|
474
670
|
return SERVICE_NUMBER.test(match.text);
|
|
475
671
|
}
|
|
476
|
-
|
|
672
|
+
// A date and the hour of a time, "07.10.2026 08" out of "07.10.2026 08:42":
|
|
673
|
+
// the French phone pattern took it as five pairs (rules-engine#101). Only when
|
|
674
|
+
// the minutes follow, so the stamp is certain.
|
|
675
|
+
const DATE_THEN_HOUR = /^\d{2}([-/.])\d{2}\1\d{4}\s+\d{1,2}$/;
|
|
676
|
+
const MINUTES_AFTER = /^:\d{2}(?!\d)/;
|
|
677
|
+
function suppressPhoneDateOverlap(text, match) {
|
|
477
678
|
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
478
679
|
return false;
|
|
479
|
-
|
|
680
|
+
const candidate = match.text.trim();
|
|
681
|
+
if (DATE_PATTERN_FULL.test(candidate))
|
|
682
|
+
return true;
|
|
683
|
+
return DATE_THEN_HOUR.test(candidate) && MINUTES_AFTER.test(text.slice(match.end, match.end + 3));
|
|
480
684
|
}
|
|
481
685
|
// Standards and classification prefixes that are plate-shaped once a letter and
|
|
482
686
|
// digits follow: "ATC-N06", "ICD-O3". None is a German district code, but the
|
|
@@ -489,6 +693,16 @@ const NOT_CITY_CODES = new Set([
|
|
|
489
693
|
"BE", "DE", "EU", "NL", "LU", "WS", "SS", "IP",
|
|
490
694
|
]);
|
|
491
695
|
const CURRENCY_PLATE = /^(?:EUR|USD|GBP|CHF|SEK|NOK|DKK|ISK|CZK|PLN|HUF|RON|BGN|HRK)\s/i;
|
|
696
|
+
const ALNUM = /[\p{L}\p{N}]/u;
|
|
697
|
+
/** Characters that join a token to more of an identifier (rules-engine#50, #81). */
|
|
698
|
+
const ID_CONNECTORS = "-/._+";
|
|
699
|
+
/** A reference marker directly before the token: "#FR-S2", "№ …", "n° …". */
|
|
700
|
+
const REFERENCE_MARK_BEFORE = /(?:#|№|(?<![A-Za-z])[nN][°º])\s?$/;
|
|
701
|
+
/** A connector glues [start, end) to a letter or digit outside it. */
|
|
702
|
+
function joinedToMore(text, start, end) {
|
|
703
|
+
return ((end + 1 < text.length && ID_CONNECTORS.includes(text[end]) && ALNUM.test(text[end + 1])) ||
|
|
704
|
+
(start >= 2 && ID_CONNECTORS.includes(text[start - 1]) && ALNUM.test(text[start - 2])));
|
|
705
|
+
}
|
|
492
706
|
function suppressPlateInCompound(text, match) {
|
|
493
707
|
if (match.patternDef.entityType !== EntityType.LICENSE_PLATE)
|
|
494
708
|
return false;
|
|
@@ -500,6 +714,18 @@ function suppressPlateInCompound(text, match) {
|
|
|
500
714
|
if (/[A-Za-zÄÖÜäöüß]{2,}-$/.test(before))
|
|
501
715
|
return true;
|
|
502
716
|
}
|
|
717
|
+
// A plate is a token of its own: joined by a connector to more of an
|
|
718
|
+
// identifier ("TF-284-KL-00874", "FR-S2/2026") it is a segment of a reference
|
|
719
|
+
// (rules-engine#50, #81). A spaced dash or slash does not join.
|
|
720
|
+
if (joinedToMore(text, match.start, match.end))
|
|
721
|
+
return true;
|
|
722
|
+
// After a reference marker ("Ref #FR-S2") it is a reference, unless a plate
|
|
723
|
+
// cue says otherwise (rules-engine#81).
|
|
724
|
+
if (REFERENCE_MARK_BEFORE.test(text.slice(Math.max(0, match.start - 4), match.start))) {
|
|
725
|
+
const [b, a] = getContext(text, match.start, match.end);
|
|
726
|
+
if (!PLATE_CUE_NEAR.test(b + a))
|
|
727
|
+
return true;
|
|
728
|
+
}
|
|
503
729
|
const matched = match.text.trim();
|
|
504
730
|
const parts = matched.split(/[\s\-]+/);
|
|
505
731
|
// A standards or classification reference, not a plate — unless a plate cue
|
|
@@ -549,7 +775,11 @@ function suppressDePlateUnknownDistrict(text, match) {
|
|
|
549
775
|
const parts = match.text.trim().split(/[\s\-]+/);
|
|
550
776
|
if (parts.length === 0 || !parts[0])
|
|
551
777
|
return false;
|
|
552
|
-
|
|
778
|
+
// A German plate number never starts with 0: without a plate cue,
|
|
779
|
+
// "GR-NB-0412" is an asset tag and "HR-IT-03" a form (rules-engine#126).
|
|
780
|
+
const digits = /\d+/.exec(match.text);
|
|
781
|
+
const leadingZero = digits !== null && digits[0].startsWith("0");
|
|
782
|
+
if (DE_DISTRICT_CODES.has(parts[0].toUpperCase()) && !leadingZero)
|
|
553
783
|
return false;
|
|
554
784
|
const [before, after] = getContext(text, match.start, match.end);
|
|
555
785
|
if (PLATE_CUE_NEAR.test(before + after))
|
|
@@ -829,10 +1059,34 @@ function suppressBicWithoutEvidence(text, match, scratch) {
|
|
|
829
1059
|
// The token itself is blanked out so it cannot vouch for itself.
|
|
830
1060
|
const unit = structuralUnit(text, match.start, match.end).split(token).join(" ".repeat(token.length));
|
|
831
1061
|
if (BIC_KEYWORD.test(unit) || IBAN_SHAPE.test(unit) || BANK_BLOCK.test(unit)) {
|
|
1062
|
+
if (bicShapedWord(text, match.start, match.end, token))
|
|
1063
|
+
return true;
|
|
832
1064
|
return occursAsLowercaseWord(text, token, scratch);
|
|
833
1065
|
}
|
|
834
1066
|
return true;
|
|
835
1067
|
}
|
|
1068
|
+
/** A personal title right before the token: "Dr. HOFFMANN", "Mme JANSSENS". */
|
|
1069
|
+
const TITLE_BEFORE = /(?<![A-Za-z])(?:dr|prof|mr|mrs|ms|mme|mlle|m|herr|frau|dhr|mevr|sig|sra?)\.?\s+$/i;
|
|
1070
|
+
const LETTER = /\p{L}/u;
|
|
1071
|
+
/** A BIC/SWIFT label touching the token: "BIC: ", "SWIFT-Code:\n", "Code SWIFT : ", "BIC Bank A: ". */
|
|
1072
|
+
const BIC_LABEL_BEFORE = /(?<![A-Za-z])(?:bic|swift)(?:[\s/\-]*(?:code|swift|bic))?(?:\s+bank\s+\w{1,3})?[\s:.\-()]*$/i;
|
|
1073
|
+
/**
|
|
1074
|
+
* A registry miss sitting where only a word or a name can (rules-engine#57):
|
|
1075
|
+
* joined to a word by a hyphen ("NGUYEN-HOFFMANN"), right after a personal
|
|
1076
|
+
* title ("Dr. HOFFMANN"), or letters only (eight, or eleven without an `XXX`
|
|
1077
|
+
* branch: "BETALING", "MAANDELIJKS") that the GLEIF mapping does not know and
|
|
1078
|
+
* no BIC/SWIFT label touches.
|
|
1079
|
+
*/
|
|
1080
|
+
function bicShapedWord(text, start, end, token) {
|
|
1081
|
+
if ((start >= 2 && text[start - 1] === "-" && LETTER.test(text[start - 2])) ||
|
|
1082
|
+
(text[end] === "-" && end + 1 < text.length && LETTER.test(text[end + 1])))
|
|
1083
|
+
return true;
|
|
1084
|
+
if (TITLE_BEFORE.test(text.slice(Math.max(0, start - 8), start)))
|
|
1085
|
+
return true;
|
|
1086
|
+
return /^[A-Za-z]+$/.test(token) && !(token.length === 11 && token.endsWith("XXX"))
|
|
1087
|
+
&& !isKnownInstitution(token)
|
|
1088
|
+
&& !BIC_LABEL_BEFORE.test(text.slice(Math.max(0, start - 30), start));
|
|
1089
|
+
}
|
|
836
1090
|
// ── Postal code: digits belonging to a longer identifier ────────────────
|
|
837
1091
|
// An identifier label immediately before the digits. A postal code is never
|
|
838
1092
|
// introduced this way; an SVNr, policy number or service number always is.
|
|
@@ -852,7 +1106,14 @@ const DIALLING_PREFIX_BEFORE = /\+\d{1,3}[\d\s\-().]*$/;
|
|
|
852
1106
|
// read as a country-prefixed address: in "PV-2026-LU-09143" the `LU` is
|
|
853
1107
|
// preceded by `-` and the German five-digit postal pattern kept `09143`
|
|
854
1108
|
// (rules-engine#31).
|
|
855
|
-
const COUNTRY_PREFIXED = /(?:^|[^A-Za-z0-9\-])[A-Z]{1,
|
|
1109
|
+
const COUNTRY_PREFIXED = /(?:^|[^A-Za-z0-9\-])([A-Z]{1,3})$/;
|
|
1110
|
+
// Postal country prefixes and the length of that country's postcode: any one or two capitals
|
|
1111
|
+
// used to pass, so "K-40817" and "B-51823" were country-prefixed postcodes (rules-engine#111).
|
|
1112
|
+
const POSTAL_PREFIX_DIGITS = { A: 4, AT: 4, B: 4, BE: 4, BG: 4, CH: 4, CY: 4, CZ: 5, D: 5, DE: 5, DK: 4, E: 5, EE: 5, ES: 5, F: 5, FI: 5, FIN: 5, FR: 5, GR: 5, H: 4, HR: 5, I: 5, IT: 5, L: 4, LT: 5, LU: 4, LV: 4, N: 4, NL: 4, NO: 4, P: 4, PL: 5, PT: 4, RO: 6, S: 5, SE: 5, SI: 4, SK: 5, SLO: 4 };
|
|
1113
|
+
function countryPrefixed(before, digits) {
|
|
1114
|
+
const m = COUNTRY_PREFIXED.exec(before);
|
|
1115
|
+
return m !== null && POSTAL_PREFIX_DIGITS[m[1]] === digits.length;
|
|
1116
|
+
}
|
|
856
1117
|
/**
|
|
857
1118
|
* Suppress bare digit runs that belong to a longer number, not an address.
|
|
858
1119
|
*
|
|
@@ -868,8 +1129,15 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
868
1129
|
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
869
1130
|
return false;
|
|
870
1131
|
const clean = match.text.trim();
|
|
871
|
-
if (!/^\d+$/.test(clean))
|
|
1132
|
+
if (!/^\d+$/.test(clean)) {
|
|
1133
|
+
// A structured form glued on by a hyphen is the tail of a reference: "CPS-HR-2026-019" (#111).
|
|
1134
|
+
const glued = text.slice(Math.max(0, match.start - 2), match.start);
|
|
1135
|
+
if (glued.length === 2 && glued[1] === "-" && /[A-Za-z0-9]/.test(glued[0])) {
|
|
1136
|
+
const m = COUNTRY_PREFIXED.exec(text.slice(0, match.start - 1));
|
|
1137
|
+
return !(m && m[1] in POSTAL_PREFIX_DIGITS);
|
|
1138
|
+
}
|
|
872
1139
|
return false;
|
|
1140
|
+
}
|
|
873
1141
|
const beforeChar = match.start > 0 ? text[match.start - 1] : "";
|
|
874
1142
|
const afterChar = match.end < text.length ? text[match.end] : "";
|
|
875
1143
|
const prevPrev = match.start >= 2 ? text[match.start - 2] : "";
|
|
@@ -888,7 +1156,10 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
888
1156
|
return true;
|
|
889
1157
|
if (afterChar === "-" && /\d/.test(nextNext))
|
|
890
1158
|
return true;
|
|
891
|
-
|
|
1159
|
+
// A hyphenated chain goes on after it: "réf. : L-2025-HD-0412" is a reference (#111).
|
|
1160
|
+
if (/^-[A-Za-z0-9]+-/.test(text.slice(match.end, match.end + 12)))
|
|
1161
|
+
return true;
|
|
1162
|
+
if (beforeChar === "-" && !countryPrefixed(text.slice(0, match.start - 1), clean))
|
|
892
1163
|
return true;
|
|
893
1164
|
// A further digit group on the same line: "1268 040390", "1234 5678 925",
|
|
894
1165
|
// "+43 664 8213 907". Horizontal whitespace only — a digit on the *next*
|
|
@@ -949,12 +1220,186 @@ function suppressRequiresContext(text, match) {
|
|
|
949
1220
|
return false;
|
|
950
1221
|
// A value filling an entire field of a delimited row is introduced by its
|
|
951
1222
|
// column, not by a word, so that is context — but only for narrow shapes.
|
|
1223
|
+
// A date's column decides when the table names it: under "début" or
|
|
1224
|
+
// "last_day" a date is not a birth date (rules-engine#126).
|
|
952
1225
|
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
953
|
-
fillsADelimitedField(text, match.start, match.end)
|
|
1226
|
+
fillsADelimitedField(text, match.start, match.end) &&
|
|
1227
|
+
!(DATE_LABEL_TYPES.has(match.patternDef.entityType) && columnHeader(text, match.start) !== null))
|
|
954
1228
|
return false;
|
|
1229
|
+
if (DATE_LABEL_TYPES.has(match.patternDef.entityType)) {
|
|
1230
|
+
return !labelsThisDate(text, match.start, match.end, match.patternDef.contextKeywords, match.patternDef.entityType);
|
|
1231
|
+
}
|
|
955
1232
|
const [before, after] = getContext(text, match.start, match.end);
|
|
956
1233
|
const context = (before + " " + after).toLowerCase();
|
|
957
|
-
|
|
1234
|
+
if (match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase())))
|
|
1235
|
+
return false;
|
|
1236
|
+
// The column names it: a passport under "Passport" rows below the header (#130).
|
|
1237
|
+
const header = columnHeader(text, match.start);
|
|
1238
|
+
return header === null || !match.patternDef.contextKeywords.some(kw => keywordAt(header, kw, false) >= 0);
|
|
1239
|
+
}
|
|
1240
|
+
/** Date types decided by a label: DOB and DATE_OF_DEATH share one date shape. */
|
|
1241
|
+
const DATE_LABEL_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
|
|
1242
|
+
/** Any common numeric date, to tell whether a label belongs to another date. */
|
|
1243
|
+
// A masked date counts as one: the gateway's rules pass reads SDK-masked text (rules-engine#111).
|
|
1244
|
+
// ... and a written-month date, now a DOB shape of its own (rules-engine#115).
|
|
1245
|
+
const ANY_DATE = new RegExp(String.raw `\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}|\[(?:dob|date_of_death)\]|(?<![A-Za-z0-9_])\d{1,2}\.?\s+(?:` + MONTH_NAMES + String.raw `)\.?\s+\d{4}(?![A-Za-z0-9_])`, "i");
|
|
1246
|
+
/**
|
|
1247
|
+
* True when one of `keywords` is this date's own label: before the date with no
|
|
1248
|
+
* other date between, or after it inside the date's own sentence. Anywhere in
|
|
1249
|
+
* the window typed every date from whichever date label the document carried
|
|
1250
|
+
* (rules-engine#52).
|
|
1251
|
+
*/
|
|
1252
|
+
/** A date followed by a list separator: an item of the list a label introduces. */
|
|
1253
|
+
const LIST_ITEM = /\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}\s*(?:,|;|–|&|\band\b|\ben\b|\bet\b|\bund\b|\bor\b|\bof\b|\bou\b|\boder\b|\brespectievelijk\b|\bbzw\.?)\s*/g;
|
|
1254
|
+
/** Punctuation and at most two short words ("e.g.", "bv.", "op") between label and date. */
|
|
1255
|
+
const LABEL_TAIL = /^[\s:.,()\-]*(?:\p{L}{1,4}(?:\.\p{L}{1,2})*\.?(?:[\s:.,()\-]+|$)){0,2}$/u;
|
|
1256
|
+
/** The birth sign "°" directly before the date, but not the number sign "n°". */
|
|
1257
|
+
const BIRTH_SIGN = /(?<![a-z])°\s?$/;
|
|
1258
|
+
/** No other date between a label and the date, except items of its list. */
|
|
1259
|
+
function nothingButAListBetween(gap) {
|
|
1260
|
+
if (!ANY_DATE.test(gap))
|
|
1261
|
+
return true;
|
|
1262
|
+
const rest = gap.replace(LIST_ITEM, " ");
|
|
1263
|
+
return !ANY_DATE.test(rest) && LABEL_TAIL.test(rest);
|
|
1264
|
+
}
|
|
1265
|
+
/**
|
|
1266
|
+
* Where `keyword` occurs in the lower-cased `haystack`, or -1. An all-capitals
|
|
1267
|
+
* keyword is an abbreviation and counts only as a word of its own: "DOB" is a
|
|
1268
|
+
* label, the "dob" in Polish "dobrowolnej" is not (rules-engine#100); a plural
|
|
1269
|
+
* "s" may follow ("DOBs e.g. …"). Other
|
|
1270
|
+
* keywords still match inside a longer word, which inflected forms need.
|
|
1271
|
+
*/
|
|
1272
|
+
function keywordAt(haystack, keyword, last) {
|
|
1273
|
+
const kw = keyword.toLowerCase();
|
|
1274
|
+
const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
|
|
1275
|
+
if (keyword !== keyword.toUpperCase() || keyword === kw) {
|
|
1276
|
+
// It must start a word, though it may run on into a longer one: "née le"
|
|
1277
|
+
// in "signée le" and "born" in "Newborn" are other words (rules-engine#126).
|
|
1278
|
+
let i = last ? haystack.lastIndexOf(kw) : haystack.indexOf(kw);
|
|
1279
|
+
while (i > 0 && isLetter(haystack[i - 1])) {
|
|
1280
|
+
i = last ? (i === 0 ? -1 : haystack.lastIndexOf(kw, i - 1)) : haystack.indexOf(kw, i + 1);
|
|
1281
|
+
}
|
|
1282
|
+
return i;
|
|
1283
|
+
}
|
|
1284
|
+
const hits = [];
|
|
1285
|
+
for (let i = haystack.indexOf(kw); i !== -1; i = haystack.indexOf(kw, i + 1)) {
|
|
1286
|
+
let next = i + kw.length;
|
|
1287
|
+
if (haystack[i - 1] === "[" && haystack[next] === "]")
|
|
1288
|
+
continue; // a placeholder ("[DOB]"), not a label (#111)
|
|
1289
|
+
if (haystack[next] === "s")
|
|
1290
|
+
next++;
|
|
1291
|
+
if (!isLetter(haystack.slice(i - 1, i)) && !isLetter(haystack.slice(next, next + 1))) {
|
|
1292
|
+
hits.push(i);
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
if (hits.length === 0)
|
|
1296
|
+
return -1;
|
|
1297
|
+
return last ? hits[hits.length - 1] : hits[0];
|
|
1298
|
+
}
|
|
1299
|
+
function labelsThisDate(text, start, end, keywords, entityType) {
|
|
1300
|
+
const [before, after] = getContext(text, start, end);
|
|
1301
|
+
const lowerBefore = before.toLowerCase();
|
|
1302
|
+
const lowerAfter = after.toLowerCase();
|
|
1303
|
+
// A label on an earlier line counts only when the date opens its own line or
|
|
1304
|
+
// the label was asked as a question (a call transcript's answer turn);
|
|
1305
|
+
// a column header is a fallback and never overrules a label of the date's own.
|
|
1306
|
+
const lineStart = lowerBefore.lastIndexOf("\n") + 1;
|
|
1307
|
+
const opensLine = lowerBefore.slice(lineStart).replace(/[ \t:|;*\-•(\[]/g, "") === "";
|
|
1308
|
+
if (entityType === EntityType.DOB && BIRTH_SIGN.test(lowerBefore))
|
|
1309
|
+
return true;
|
|
1310
|
+
for (const keyword of keywords) {
|
|
1311
|
+
const kw = keyword.toLowerCase();
|
|
1312
|
+
const i = keywordAt(lowerBefore, keyword, true);
|
|
1313
|
+
if (i >= 0 && nothingButAListBetween(lowerBefore.slice(i + kw.length))
|
|
1314
|
+
&& (i >= lineStart || opensLine
|
|
1315
|
+
|| lowerBefore.slice(i + kw.length, lineStart).includes("?")))
|
|
1316
|
+
return true;
|
|
1317
|
+
const j = keywordAt(lowerAfter, keyword, false);
|
|
1318
|
+
if (j >= 0) {
|
|
1319
|
+
const gap = lowerAfter.slice(0, j);
|
|
1320
|
+
if (!ANY_DATE.test(gap) && !SENTENCE_BREAK.test(gap)
|
|
1321
|
+
&& !LEADS_TO_A_DATE.test(lowerAfter.slice(j + kw.length)))
|
|
1322
|
+
return true;
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
const header = columnHeader(text, start);
|
|
1326
|
+
return header !== null && keywords.some(kw => keywordAt(header, kw, false) >= 0);
|
|
1327
|
+
}
|
|
1328
|
+
/**
|
|
1329
|
+
* A label running straight into a date of its own introduces that date, not the
|
|
1330
|
+
* one before it ("Verstorben am 01.02.2020, geboren am 12.03.1940").
|
|
1331
|
+
*/
|
|
1332
|
+
// ... or into a value already masked ("né le [DOB]"): the label is that value's (rules-engine#111).
|
|
1333
|
+
const LEADS_TO_A_DATE = new RegExp(String.raw `^[\s:.\-]*(?:[a-zà-ÿ]{1,3}[\s:.]+)?(?:\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}|\[[a-z_]+\]|\d{1,2}\.?\s+(?:` + MONTH_NAMES + String.raw `)\.?\s+\d{4})`, "i");
|
|
1334
|
+
/** Field separators a table row may use. */
|
|
1335
|
+
const COLUMN_SEPARATORS = [";", "|", "\t", ","];
|
|
1336
|
+
/**
|
|
1337
|
+
* The lower-cased header cell above the value, if it sits in a table: in a table
|
|
1338
|
+
* the column decides the label, not proximity. A header is one of the three
|
|
1339
|
+
* lines above, split by the row's separator into the same number of cells.
|
|
1340
|
+
*/
|
|
1341
|
+
function columnHeader(text, start) {
|
|
1342
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
1343
|
+
const lineEndAt = text.indexOf("\n", start);
|
|
1344
|
+
const line = text.slice(lineStart, lineEndAt >= 0 ? lineEndAt : text.length);
|
|
1345
|
+
for (const sep of COLUMN_SEPARATORS) {
|
|
1346
|
+
if (!line.includes(sep))
|
|
1347
|
+
continue;
|
|
1348
|
+
const cells = line.split(sep);
|
|
1349
|
+
const column = text.slice(lineStart, start).split(sep).length - 1;
|
|
1350
|
+
// Lines above, one lastIndexOf at a time: one of the three lines above, or
|
|
1351
|
+
// further up through rows of the same table (rules-engine#126).
|
|
1352
|
+
let end = lineStart - 1;
|
|
1353
|
+
let seen = 0;
|
|
1354
|
+
while (end >= 0 && seen < HEADER_REACH) {
|
|
1355
|
+
const begin = end === 0 ? 0 : text.lastIndexOf("\n", end - 1) + 1;
|
|
1356
|
+
const header = text.slice(begin, end);
|
|
1357
|
+
seen++;
|
|
1358
|
+
// A rule line ("|---|---|", "----+----") is neither header nor row (#130).
|
|
1359
|
+
if (RULE_LINE.test(header)) {
|
|
1360
|
+
end = begin - 1;
|
|
1361
|
+
continue;
|
|
1362
|
+
}
|
|
1363
|
+
const heads = header.split(sep);
|
|
1364
|
+
const same = heads.length === cells.length;
|
|
1365
|
+
// A "label : value" field with a comma in its value is not a header
|
|
1366
|
+
// (#111); a commented header ("# Spalten: Name;von;bis") is (#126).
|
|
1367
|
+
// Nor is a data row without a date: a header has no cell with four digits or more (#130).
|
|
1368
|
+
if (same && (!header.includes(":") || header.trimStart().startsWith("#")) && !ANY_DATE.test(header)
|
|
1369
|
+
&& !heads.some((h) => (h.match(/\d/g) ?? []).length >= 4)) {
|
|
1370
|
+
return heads[column].toLowerCase();
|
|
1371
|
+
}
|
|
1372
|
+
if (seen >= 3 && !same)
|
|
1373
|
+
break; // past the table this value sits in
|
|
1374
|
+
end = begin - 1;
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
return null;
|
|
1378
|
+
}
|
|
1379
|
+
/** How many lines up a table's header is looked for. */
|
|
1380
|
+
const HEADER_REACH = 200;
|
|
1381
|
+
/** A table's rule line: only dashes, equals signs, plus signs, colons, pipes. */
|
|
1382
|
+
const RULE_LINE = /^[\s\-=+:|]*-[\s\-=+:|]*$/;
|
|
1383
|
+
/** A dotted MAC shape glued to more of a reference is part of it: the tail of
|
|
1384
|
+
* "WSO-II.6151.4471.2025" (rules-engine#105). Dotted form only. */
|
|
1385
|
+
function suppressMacInReference(text, match) {
|
|
1386
|
+
if (match.patternDef.entityType !== EntityType.MAC_ADDRESS || !match.text.includes("."))
|
|
1387
|
+
return false;
|
|
1388
|
+
return joinedToMore(text, match.start, match.end);
|
|
1389
|
+
}
|
|
1390
|
+
/** A card number is never carved out of a longer run of digit groups: a
|
|
1391
|
+
* four-digit group joined by the card's own separator on either side means the
|
|
1392
|
+
* run is longer, e.g. a Polish account number (rules-engine#106). */
|
|
1393
|
+
function suppressCardInsideDigitRun(text, match) {
|
|
1394
|
+
if (match.patternDef.entityType !== EntityType.CREDIT_CARD)
|
|
1395
|
+
return false;
|
|
1396
|
+
const sep = [...match.text].find(c => !/\d/.test(c)) ?? "";
|
|
1397
|
+
if (!sep)
|
|
1398
|
+
return false;
|
|
1399
|
+
const s = sep.replace(/[-\\^$.*+?()[\]{}|]/g, "\\$&");
|
|
1400
|
+
const before = text.slice(Math.max(0, match.start - 6), match.start);
|
|
1401
|
+
const after = text.slice(match.end, match.end + 6);
|
|
1402
|
+
return new RegExp(`(?<!\\d)\\d{4}${s}$`).test(before) || new RegExp(`^${s}\\d{4}(?!\\d)`).test(after);
|
|
958
1403
|
}
|
|
959
1404
|
const TYPE_SUPPRESSORS = {
|
|
960
1405
|
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneServiceNumber, suppressPhoneInsideAccountRun, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
@@ -967,12 +1412,20 @@ const TYPE_SUPPRESSORS = {
|
|
|
967
1412
|
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
968
1413
|
[EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
969
1414
|
[EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
1415
|
+
[EntityType.MAC_ADDRESS]: [suppressMacInReference],
|
|
1416
|
+
[EntityType.CREDIT_CARD]: [suppressCardInsideDigitRun],
|
|
1417
|
+
[EntityType.IP_ADDRESS]: [suppressIpNotAHost],
|
|
970
1418
|
};
|
|
1419
|
+
for (const type of EXAMPLE_TYPES) {
|
|
1420
|
+
TYPE_SUPPRESSORS[type] = [...(TYPE_SUPPRESSORS[type] ?? []), suppressExampleValue, suppressNotAnIdentifier];
|
|
1421
|
+
}
|
|
971
1422
|
export function shouldSuppress(text, match, scratch) {
|
|
972
1423
|
if (suppressSequential(text, match))
|
|
973
1424
|
return true;
|
|
974
1425
|
if (suppressRedactionPlaceholder(text, match))
|
|
975
1426
|
return true;
|
|
1427
|
+
if (suppressDummyValue(text, match))
|
|
1428
|
+
return true;
|
|
976
1429
|
const typeSups = TYPE_SUPPRESSORS[match.patternDef.entityType];
|
|
977
1430
|
if (typeSups) {
|
|
978
1431
|
for (const s of typeSups) {
|