euredact 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +658 -45
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +131 -18
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +138 -48
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +418 -23
- package/dist/cjs/rules/validators.js +103 -0
- package/dist/cjs/sdk.js +150 -44
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +90 -5
- package/dist/esm/cloud/client.js +126 -26
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +138 -48
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +417 -22
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +100 -0
- package/dist/esm/sdk.d.ts +63 -19
- package/dist/esm/sdk.js +148 -44
- package/dist/esm/types.d.ts +58 -2
- package/dist/esm/types.js +4 -1
- package/package.json +12 -4
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SuppressionScratch = void 0;
|
|
3
|
+
exports.SuppressionScratch = exports.MONTH_NAMES = void 0;
|
|
4
4
|
exports.shouldSuppress = shouldSuppress;
|
|
5
5
|
const types_js_1 = require("../types.js");
|
|
6
6
|
const bicRegistry_js_1 = require("./bicRegistry.js");
|
|
@@ -25,10 +25,11 @@ const CURRENCY_WORD = "EUR|USD|GBP|CHF|ISK|SEK|NOK|DKK|PLN|CZK|HUF|RON|BGN|HRK|"
|
|
|
25
25
|
const CURRENCY_AFTER = new RegExp(`^\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
26
26
|
const CURRENCY_COMMA_AFTER = new RegExp(`^[.,]\\d{1,2}\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
27
27
|
const CURRENCY_BEFORE = new RegExp(`(?:(?:${CURRENCY_SYMBOL})|\\b(?:${CURRENCY_WORD}))\\s*$`, "i");
|
|
28
|
-
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)
|
|
28
|
+
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)(?:\s+\w+)?\s*:?\s*$/i; // second word separate: not "Rent"+"enversicherungsnummer" (rules-engine#111)
|
|
29
29
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
30
30
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
31
|
-
|
|
31
|
+
// Horizontal space only: a unit is on the number's own line (rules-engine#51).
|
|
32
|
+
const UNIT_AFTER = /^[ \t]*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
32
33
|
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
33
34
|
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|ticket|incident\s*(?:report|nr|no)?|case\s*(?:nr|no|id)?|zaaknummer|meldingsnummer|Vorgangsnummer|Vorgang|Störungsmeldung|saksnummer|ärendenummer|sagsnummer|asianumero|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.#]?\s*$/i;
|
|
34
35
|
// "#" before a number marks a reference in every language in scope — a ticket,
|
|
@@ -36,9 +37,19 @@ const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|f
|
|
|
36
37
|
const HASH_BEFORE = /#\s*$/;
|
|
37
38
|
// A short uppercase tag hyphenated to the number: "IR-43433", "INC-2024".
|
|
38
39
|
const REF_PREFIX_BEFORE = /(?:^|[\s([])[A-Z]{2,5}-$/;
|
|
40
|
+
// The cross-border address form, "CH-8004 Zürich": a country code hyphenated to
|
|
41
|
+
// a postal code, told apart from a reference tag by position -- it opens an
|
|
42
|
+
// address line and a place name follows (rules-engine#58).
|
|
43
|
+
const COUNTRY_PREFIX_OPENS_LINE = /(?:^|\n|,)[ \t]*([A-Z]{2})-$/;
|
|
44
|
+
const POSTAL_PREFIX_COUNTRIES = new Set([
|
|
45
|
+
"AT", "BE", "BG", "CH", "CY", "CZ", "DE", "DK", "EE", "EL", "ES", "FI",
|
|
46
|
+
"FR", "GB", "GR", "HR", "HU", "IE", "IS", "IT", "LI", "LT", "LU", "LV",
|
|
47
|
+
"MT", "NL", "NO", "PL", "PT", "RO", "SE", "SI", "SK", "UK",
|
|
48
|
+
]);
|
|
39
49
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
40
|
-
|
|
41
|
-
const
|
|
50
|
+
// An operator on the same line only: a bullet or "---" on the next line is not a minus sign (rules-engine#91).
|
|
51
|
+
const MATH_BEFORE = /[=+\-×÷*/][^\S\r\n]*$/;
|
|
52
|
+
const MATH_AFTER = /^[^\S\r\n]*[=+\-×÷*/]/;
|
|
42
53
|
// "A-1010", "B-2000", "L-1234", "CH-8000", "D-10115": a country prefix on a
|
|
43
54
|
// postal code, not a minus sign.
|
|
44
55
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
@@ -64,10 +75,11 @@ const NUMERIC_TYPES = new Set([types_js_1.EntityType.PHONE, types_js_1.EntityTyp
|
|
|
64
75
|
// Month names in the languages this engine covers. Long, but it is data: the
|
|
65
76
|
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
66
77
|
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
67
|
-
|
|
78
|
+
exports.MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
68
79
|
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
69
80
|
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
70
81
|
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
82
|
+
"|janv|févr|fevr|avr|juil|déc" +
|
|
71
83
|
"|januari|februari|maart|mei|augustus|mrt" +
|
|
72
84
|
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
73
85
|
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
@@ -94,7 +106,7 @@ const SINCE_CUES = "seit|since|depuis|sinds|sedert|vanaf|desde|dal|od|sedan|side
|
|
|
94
106
|
* candidate. Adjacent-only on purpose — a cue 150 characters away is what made
|
|
95
107
|
* this a bug in the first place.
|
|
96
108
|
*/
|
|
97
|
-
const DATE_BEFORE = new RegExp(`(?:\\b(?:${MONTH_NAMES})\\b\\.?|\\b(?:${SINCE_CUES})\\b|\\d{1,2}[./-]\\d{1,2}[./-])\\s*$`, "iu");
|
|
109
|
+
const DATE_BEFORE = new RegExp(`(?:\\b(?:${exports.MONTH_NAMES})\\b\\.?|\\b(?:${SINCE_CUES})\\b|\\d{1,2}[./-]\\d{1,2}[./-])\\s*$`, "iu");
|
|
98
110
|
/** A date separator immediately after, making the candidate a leading year. */
|
|
99
111
|
const DATE_AFTER = /^[./-]\d{1,2}[./-]\d{1,2}\b/;
|
|
100
112
|
/**
|
|
@@ -289,7 +301,19 @@ function suppressReference(text, match) {
|
|
|
289
301
|
if (REFERENCE_BEFORE.test(before))
|
|
290
302
|
return true;
|
|
291
303
|
const adjacent = text.slice(Math.max(0, match.start - 8), match.start);
|
|
292
|
-
|
|
304
|
+
if (HASH_BEFORE.test(adjacent))
|
|
305
|
+
return true;
|
|
306
|
+
if (!REF_PREFIX_BEFORE.test(adjacent))
|
|
307
|
+
return false;
|
|
308
|
+
return !(match.patternDef.entityType === types_js_1.EntityType.POSTAL_CODE &&
|
|
309
|
+
isCountryPrefixedAddress(text, match.start, match.end));
|
|
310
|
+
}
|
|
311
|
+
/** "CH-8004 Zürich" on an address line, not the reference "IT-20431". */
|
|
312
|
+
function isCountryPrefixedAddress(text, start, end) {
|
|
313
|
+
const found = COUNTRY_PREFIX_OPENS_LINE.exec(text.slice(Math.max(0, start - 12), start));
|
|
314
|
+
return (found !== null &&
|
|
315
|
+
POSTAL_PREFIX_COUNTRIES.has(found[1]) &&
|
|
316
|
+
CITY_AFTER.test(text.slice(end, end + 24)));
|
|
293
317
|
}
|
|
294
318
|
function suppressLegal(text, match) {
|
|
295
319
|
const applicable = LEGAL_TYPES;
|
|
@@ -313,14 +337,105 @@ function suppressSequential(_text, match) {
|
|
|
313
337
|
const clean = match.text.replace(/[\s.\-]/g, "");
|
|
314
338
|
return SEQUENTIAL_PATTERNS.test(clean);
|
|
315
339
|
}
|
|
340
|
+
// ── The engine's own output ───────────────────────────────────────────────
|
|
341
|
+
//
|
|
342
|
+
// Redaction has to be idempotent: running it over a document it already
|
|
343
|
+
// produced must leave the placeholders it wrote alone. It did not.
|
|
344
|
+
// `[POSTAL_CODE]` is thirteen characters of mixed case with an underscore, so
|
|
345
|
+
// the entropy-based SECRET rule read it as a credential and a second pass
|
|
346
|
+
// replaced it with `[SECRET]` (rules-engine#33).
|
|
347
|
+
//
|
|
348
|
+
// All three emitted forms are guarded, because all three come back as input in
|
|
349
|
+
// a re-processing pipeline: `[TYPE]`, `TYPE_1` (referentialIntegrity) and
|
|
350
|
+
// `TYPE_K7Q2` (tokenize -- where a false detection breaks restore()).
|
|
351
|
+
//
|
|
352
|
+
// Only real entity-type names count. Guarding any bracketed upper-case token
|
|
353
|
+
// would be simpler and wrong: `[AKIAIOSFODNN7EXAMPLE]` is one, and also a live
|
|
354
|
+
// AWS key. A placeholder for a *custom* pattern is therefore not covered.
|
|
355
|
+
// The closing bracket is optional: another pattern's span need not agree with
|
|
356
|
+
// this one about where the marker ends. The assigned-secret rule stops before
|
|
357
|
+
// sentence punctuation (rules-engine#35), so it claims "[POSTAL_CODE" without
|
|
358
|
+
// the "]". The opening bracket is still required.
|
|
359
|
+
const PLACEHOLDER = /^(?:\[([A-Z][A-Z0-9_]*)\]?|([A-Z][A-Z0-9_]*?)_(?:\d+|[A-HJ-NP-Z2-9]{4}))$/;
|
|
360
|
+
const KNOWN_TYPE_NAMES = new Set(Object.values(types_js_1.EntityType));
|
|
361
|
+
function suppressRedactionPlaceholder(_text, match) {
|
|
362
|
+
const found = PLACEHOLDER.exec(match.text);
|
|
363
|
+
if (found === null)
|
|
364
|
+
return false;
|
|
365
|
+
const name = found[1] ?? found[2];
|
|
366
|
+
return KNOWN_TYPE_NAMES.has(name);
|
|
367
|
+
}
|
|
316
368
|
const RECENT_YEAR = /^(?:19[5-9]\d|20[0-3]\d)$/;
|
|
369
|
+
// Any year a written date can carry: the date-construction test applies to all of them (rules-engine#111).
|
|
370
|
+
const DATE_YEAR = /^(?:1[89]\d\d|20\d\d)$/;
|
|
317
371
|
const POSTAL_CONTEXT_NEAR = /(?:postcode|postal|code\s*postal|PLZ|Postleitzahl|postnummer|postinumero|póstnúmer|zip|straat|straße|strasse|rue\s|via\s|calle\s|rua\s|ulica|utca|street|avenue|laan\s|weg\s|plein|adres|adresse|address|woonplaats|wonende|woonachtig|gevestigd|domicili|demeurant|résidant|residant|bosatt|bopæl|wohnhaft|ansässig|stad\b|ville\b|city\b|Stadt|città|ciudad|cidade|miasto|město|város)/i;
|
|
318
372
|
const DATE_KEYWORD_NEAR = /(?:født|fødselsdato|fødsel|Fødselsdato|född|födelsedatum|födelsedag|syntynyt|syntymäaika|fæddur|fæðingardagur|geboren|geboortedatum|Geburtsdatum|nascido|nacido|data di nascita|nato il|nata il|Tiltr[æa]delsesdato|Tiltredelsesdato|datum|date\b|Datum|jaar|year|année|Jahr|since|sinds|depuis|seit|\d{2}\.\d{2}\.|(?:januar|februar|marts|april|maj|juni|juli|august|september|oktober|november|december|januari|februari|mars|mei|juin|juillet|août))/i;
|
|
373
|
+
// A sentence boundary: a full stop, question or exclamation mark followed by
|
|
374
|
+
// space, or a line break. Written so it cannot fire inside "11.03.1970", where
|
|
375
|
+
// the periods are not followed by whitespace -- that date is one of the false
|
|
376
|
+
// positives this gate has to reject.
|
|
377
|
+
const SENTENCE_BREAK = /[.!?][ \t\n]|\n/;
|
|
378
|
+
const SENTENCE_BREAK_G = /[.!?][ \t\n]|\n/g;
|
|
379
|
+
// The value's own sentence, bounded either side.
|
|
380
|
+
function sentenceAround(text, start, end, reach = 160) {
|
|
381
|
+
let before = text.slice(Math.max(0, start - reach), start);
|
|
382
|
+
let after = text.slice(end, end + reach);
|
|
383
|
+
let lastEnd = -1;
|
|
384
|
+
SENTENCE_BREAK_G.lastIndex = 0;
|
|
385
|
+
for (let m = SENTENCE_BREAK_G.exec(before); m !== null; m = SENTENCE_BREAK_G.exec(before)) {
|
|
386
|
+
lastEnd = m.index + m[0].length;
|
|
387
|
+
}
|
|
388
|
+
if (lastEnd >= 0)
|
|
389
|
+
before = before.slice(lastEnd);
|
|
390
|
+
const found = SENTENCE_BREAK.exec(after);
|
|
391
|
+
if (found !== null)
|
|
392
|
+
after = after.slice(0, found.index);
|
|
393
|
+
return before + after;
|
|
394
|
+
}
|
|
395
|
+
// A postal label whose only distance from the value is punctuation:
|
|
396
|
+
// "PLZ: 2011", "code postal : 2011", "Woonplaats: 1950".
|
|
397
|
+
const POSTAL_LABEL_TOUCHING = new RegExp(`(?:${POSTAL_CONTEXT_NEAR.source})[\\s:.\\-]{0,4}$`, "i");
|
|
398
|
+
// A capitalised place name after the code: "2000 Antwerpen". A following label
|
|
399
|
+
// ("1970 Fødselsnummer:") is excluded by the colon.
|
|
400
|
+
// `\p{L}\p{N}_` rather than `\w`: JS `\w` is ASCII-only, so "8004 Zürich" failed
|
|
401
|
+
// at the ü here while Python's Unicode `\w` accepted it (rules-engine#58).
|
|
402
|
+
const CITY_AFTER = /^[ \t](?![\p{L}\p{N}_]+\s*:)[A-ZÀ-ÞŁŠŽ][\p{L}\p{N}_\-']{2,}/u;
|
|
403
|
+
// Every position is local to the value or inside its own sentence. Proximity to
|
|
404
|
+
// an address *word* anywhere in a 300-character window is what the old rescue
|
|
405
|
+
// asked, and that is a neighbourhood rather than a position (rules-engine#32).
|
|
406
|
+
/**
|
|
407
|
+
* The comma of an address line: the word before it is a place name or a number.
|
|
408
|
+
* "myocardial infarction, 2016" is a year in a list (rules-engine#111).
|
|
409
|
+
*/
|
|
410
|
+
const ADDRESS_COMMA = /(?<!\S)(?:[A-ZÀ-Þ\d][^\s,]*|[^\s,]*\]),[ \t]*$/;
|
|
411
|
+
/**
|
|
412
|
+
* A redaction placeholder: "[POSTAL_CODE]" holds the words the address test
|
|
413
|
+
* looks for, so on masked text it made every year in its sentence a postcode
|
|
414
|
+
* (rules-engine#111). A masked value is no address word.
|
|
415
|
+
*/
|
|
416
|
+
const MASKED_VALUE = /\[[A-Z][A-Z_]*\]/g;
|
|
417
|
+
function yearSitsInAddressStructure(text, start, end) {
|
|
418
|
+
const before = text.slice(Math.max(0, start - 28), start);
|
|
419
|
+
// "Amsterdam, 2026" -- the comma of an address line.
|
|
420
|
+
if (ADDRESS_COMMA.test(before))
|
|
421
|
+
return true;
|
|
422
|
+
// "PLZ: 2011" -- a postal label with nothing but punctuation between.
|
|
423
|
+
if (POSTAL_LABEL_TOUCHING.test(before))
|
|
424
|
+
return true;
|
|
425
|
+
// A capitalised place name immediately after.
|
|
426
|
+
if (CITY_AFTER.test(text.slice(end, end + 24)))
|
|
427
|
+
return true;
|
|
428
|
+
// An address word in the value's own sentence. "Te huur: Lange Nieuwstraat
|
|
429
|
+
// 12, rustige ligging in 2018, vlakbij openbaar vervoer." is an Antwerp
|
|
430
|
+
// postal code in prose; the false positives put the year and the address in
|
|
431
|
+
// different sentences.
|
|
432
|
+
return POSTAL_CONTEXT_NEAR.test(sentenceAround(text, start, end).replace(MASKED_VALUE, " "));
|
|
433
|
+
}
|
|
319
434
|
function suppressYearAsPostal(text, match) {
|
|
320
435
|
if (match.patternDef.entityType !== types_js_1.EntityType.POSTAL_CODE)
|
|
321
436
|
return false;
|
|
322
437
|
const clean = match.text.trim();
|
|
323
|
-
if (!
|
|
438
|
+
if (!DATE_YEAR.test(clean))
|
|
324
439
|
return false;
|
|
325
440
|
// A date construction *touching* the candidate settles it, whatever the rest
|
|
326
441
|
// of the document says. Without this, the postal-context test below rescued
|
|
@@ -332,14 +447,14 @@ function suppressYearAsPostal(text, match) {
|
|
|
332
447
|
return true;
|
|
333
448
|
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
334
449
|
return true;
|
|
450
|
+
if (!RECENT_YEAR.test(clean))
|
|
451
|
+
return false;
|
|
335
452
|
const [before, after] = getContext(text, match.start, match.end);
|
|
336
453
|
const context = before + after;
|
|
337
|
-
// Keep as postal code
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
const immediateBefore = text.slice(Math.max(0, match.start - 3), match.start);
|
|
342
|
-
if (/,\s*$/.test(immediateBefore))
|
|
454
|
+
// Keep as postal code only when the value sits in address *structure* --
|
|
455
|
+
// see yearSitsInAddressStructure for why a 300-character window of address
|
|
456
|
+
// words was the defect rather than the keyword list.
|
|
457
|
+
if (yearSitsInAddressStructure(text, match.start, match.end))
|
|
343
458
|
return false;
|
|
344
459
|
// Suppress if date keyword nearby (birth date, employment date, etc.)
|
|
345
460
|
if (DATE_KEYWORD_NEAR.test(context))
|
|
@@ -383,12 +498,26 @@ const TOWN_AFTER = /^\s*[A-ZÀ-ÞĀ-Ž]/;
|
|
|
383
498
|
* between the marker and the number, so allow exactly one.
|
|
384
499
|
*/
|
|
385
500
|
const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|durchwahl|doorkiesnummer|tst|nebenstelle)\.?\s*(?:[a-zà-ÿ]{2,10}\s*)?[:.]?\s*$/i;
|
|
501
|
+
/**
|
|
502
|
+
* What else a bare digit run after these is (rules-engine#111): a head count
|
|
503
|
+
* ("Employees: 214"), a standard ("ISO/IEC 27001"), a product model with an
|
|
504
|
+
* inner capital ("Dell OptiPlex 7090"). Not the digits after "[PHONE]": the
|
|
505
|
+
* rest of a number an earlier pass cut short, safer masked as a postcode.
|
|
506
|
+
*/
|
|
507
|
+
const COUNT_LABEL_BEFORE = /(?<![A-Za-z0-9_])(?:employees|headcount|staff|mitarbeiter(?:innen)?|beschäftigte|werknemers|medewerkers|employés|salariés|effectifs?|dipendenti|empleados|pracowników|zaměstnanců)\s*[:=]?\s*$/i;
|
|
508
|
+
const NOT_POSTAL_BEFORE = /(?<![A-Za-z0-9_])(?:(?:ISO(?:\/IEC)?|IEC|EN|DIN(?:\s+EN)?|NEN(?:-EN)?|NF\s+EN|BS(?:\s+EN)?|PN-EN|ÖNORM)[\s\-]*|[A-Za-z]*[a-z][A-Z][A-Za-z]*\s+)$/;
|
|
509
|
+
/** A thousands group of an amount: the "750" of "£28,750,000". */
|
|
510
|
+
const GROUP_BEFORE = /\d[,.]$/;
|
|
511
|
+
const GROUP_AFTER = /^[,.]\d{3}(?!\d)/;
|
|
386
512
|
/** Suppress a postal code the word in front of it rules out. */
|
|
387
513
|
function suppressPostalAfterDisqualifyingWord(text, match) {
|
|
388
514
|
if (match.patternDef.entityType !== types_js_1.EntityType.POSTAL_CODE)
|
|
389
515
|
return false;
|
|
390
516
|
const before = text.slice(Math.max(0, match.start - 24), match.start);
|
|
391
|
-
if (EXTENSION_BEFORE.test(before))
|
|
517
|
+
if (EXTENSION_BEFORE.test(before) || COUNT_LABEL_BEFORE.test(before)
|
|
518
|
+
|| NOT_POSTAL_BEFORE.test(before))
|
|
519
|
+
return true;
|
|
520
|
+
if (GROUP_BEFORE.test(before) && GROUP_AFTER.test(text.slice(match.end, match.end + 5)))
|
|
392
521
|
return true;
|
|
393
522
|
return RECENT_YEAR.test(match.text.trim())
|
|
394
523
|
&& (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
|
|
@@ -399,10 +528,18 @@ function suppressPhoneServiceNumber(_text, match) {
|
|
|
399
528
|
return false;
|
|
400
529
|
return SERVICE_NUMBER.test(match.text);
|
|
401
530
|
}
|
|
402
|
-
|
|
531
|
+
// A date and the hour of a time, "07.10.2026 08" out of "07.10.2026 08:42":
|
|
532
|
+
// the French phone pattern took it as five pairs (rules-engine#101). Only when
|
|
533
|
+
// the minutes follow, so the stamp is certain.
|
|
534
|
+
const DATE_THEN_HOUR = /^\d{2}([-/.])\d{2}\1\d{4}\s+\d{1,2}$/;
|
|
535
|
+
const MINUTES_AFTER = /^:\d{2}(?!\d)/;
|
|
536
|
+
function suppressPhoneDateOverlap(text, match) {
|
|
403
537
|
if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
|
|
404
538
|
return false;
|
|
405
|
-
|
|
539
|
+
const candidate = match.text.trim();
|
|
540
|
+
if (DATE_PATTERN_FULL.test(candidate))
|
|
541
|
+
return true;
|
|
542
|
+
return DATE_THEN_HOUR.test(candidate) && MINUTES_AFTER.test(text.slice(match.end, match.end + 3));
|
|
406
543
|
}
|
|
407
544
|
// Standards and classification prefixes that are plate-shaped once a letter and
|
|
408
545
|
// digits follow: "ATC-N06", "ICD-O3". None is a German district code, but the
|
|
@@ -415,6 +552,16 @@ const NOT_CITY_CODES = new Set([
|
|
|
415
552
|
"BE", "DE", "EU", "NL", "LU", "WS", "SS", "IP",
|
|
416
553
|
]);
|
|
417
554
|
const CURRENCY_PLATE = /^(?:EUR|USD|GBP|CHF|SEK|NOK|DKK|ISK|CZK|PLN|HUF|RON|BGN|HRK)\s/i;
|
|
555
|
+
const ALNUM = /[\p{L}\p{N}]/u;
|
|
556
|
+
/** Characters that join a token to more of an identifier (rules-engine#50, #81). */
|
|
557
|
+
const ID_CONNECTORS = "-/._+";
|
|
558
|
+
/** A reference marker directly before the token: "#FR-S2", "№ …", "n° …". */
|
|
559
|
+
const REFERENCE_MARK_BEFORE = /(?:#|№|(?<![A-Za-z])[nN][°º])\s?$/;
|
|
560
|
+
/** A connector glues [start, end) to a letter or digit outside it. */
|
|
561
|
+
function joinedToMore(text, start, end) {
|
|
562
|
+
return ((end + 1 < text.length && ID_CONNECTORS.includes(text[end]) && ALNUM.test(text[end + 1])) ||
|
|
563
|
+
(start >= 2 && ID_CONNECTORS.includes(text[start - 1]) && ALNUM.test(text[start - 2])));
|
|
564
|
+
}
|
|
418
565
|
function suppressPlateInCompound(text, match) {
|
|
419
566
|
if (match.patternDef.entityType !== types_js_1.EntityType.LICENSE_PLATE)
|
|
420
567
|
return false;
|
|
@@ -426,6 +573,18 @@ function suppressPlateInCompound(text, match) {
|
|
|
426
573
|
if (/[A-Za-zÄÖÜäöüß]{2,}-$/.test(before))
|
|
427
574
|
return true;
|
|
428
575
|
}
|
|
576
|
+
// A plate is a token of its own: joined by a connector to more of an
|
|
577
|
+
// identifier ("TF-284-KL-00874", "FR-S2/2026") it is a segment of a reference
|
|
578
|
+
// (rules-engine#50, #81). A spaced dash or slash does not join.
|
|
579
|
+
if (joinedToMore(text, match.start, match.end))
|
|
580
|
+
return true;
|
|
581
|
+
// After a reference marker ("Ref #FR-S2") it is a reference, unless a plate
|
|
582
|
+
// cue says otherwise (rules-engine#81).
|
|
583
|
+
if (REFERENCE_MARK_BEFORE.test(text.slice(Math.max(0, match.start - 4), match.start))) {
|
|
584
|
+
const [b, a] = getContext(text, match.start, match.end);
|
|
585
|
+
if (!PLATE_CUE_NEAR.test(b + a))
|
|
586
|
+
return true;
|
|
587
|
+
}
|
|
429
588
|
const matched = match.text.trim();
|
|
430
589
|
const parts = matched.split(/[\s\-]+/);
|
|
431
590
|
// A standards or classification reference, not a plate — unless a plate cue
|
|
@@ -510,6 +669,35 @@ function suppressPostalInsideIban(text, match) {
|
|
|
510
669
|
}
|
|
511
670
|
return false;
|
|
512
671
|
}
|
|
672
|
+
// An IBAN-shaped run: two letters, two check digits, then grouped
|
|
673
|
+
// alphanumerics. Deliberately not checksum-gated -- the point is to describe the
|
|
674
|
+
// run a human wrote as one account number, including the ones that fail their
|
|
675
|
+
// checksum, because those are exactly the ones no structured detector claims.
|
|
676
|
+
const ACCOUNT_RUN = /\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{2,4}){2,8}\b/g;
|
|
677
|
+
// "Compte: FR76 3000 4008 0300 0109 5374" fails the IBAN checksum, so no
|
|
678
|
+
// BANK_ACCOUNT candidate survives and the Danish eight-digit phone pattern took
|
|
679
|
+
// "3000 4008" out of the middle of it, leaving the rest in the clear
|
|
680
|
+
// (rules-engine#30). POSTAL_CODE has carried its own version of this guard
|
|
681
|
+
// since 0.3.3, in suppressPostalInsideIban.
|
|
682
|
+
function suppressPhoneInsideAccountRun(text, match) {
|
|
683
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
|
|
684
|
+
return false;
|
|
685
|
+
const lineStart = text.lastIndexOf("\n", match.start) + 1;
|
|
686
|
+
let lineEnd = text.indexOf("\n", match.end);
|
|
687
|
+
if (lineEnd < 0)
|
|
688
|
+
lineEnd = text.length;
|
|
689
|
+
const line = text.slice(lineStart, lineEnd);
|
|
690
|
+
ACCOUNT_RUN.lastIndex = 0;
|
|
691
|
+
for (let m = ACCOUNT_RUN.exec(line); m !== null; m = ACCOUNT_RUN.exec(line)) {
|
|
692
|
+
const runStart = lineStart + m.index;
|
|
693
|
+
const runEnd = runStart + m[0].length;
|
|
694
|
+
if (runStart <= match.start && match.end <= runEnd
|
|
695
|
+
&& runEnd - runStart > match.end - match.start) {
|
|
696
|
+
return true;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return false;
|
|
700
|
+
}
|
|
513
701
|
function suppressPostalAsHouseNumber(text, match) {
|
|
514
702
|
if (match.patternDef.entityType !== types_js_1.EntityType.POSTAL_CODE)
|
|
515
703
|
return false;
|
|
@@ -727,10 +915,34 @@ function suppressBicWithoutEvidence(text, match, scratch) {
|
|
|
727
915
|
// The token itself is blanked out so it cannot vouch for itself.
|
|
728
916
|
const unit = structuralUnit(text, match.start, match.end).split(token).join(" ".repeat(token.length));
|
|
729
917
|
if (BIC_KEYWORD.test(unit) || IBAN_SHAPE.test(unit) || BANK_BLOCK.test(unit)) {
|
|
918
|
+
if (bicShapedWord(text, match.start, match.end, token))
|
|
919
|
+
return true;
|
|
730
920
|
return occursAsLowercaseWord(text, token, scratch);
|
|
731
921
|
}
|
|
732
922
|
return true;
|
|
733
923
|
}
|
|
924
|
+
/** A personal title right before the token: "Dr. HOFFMANN", "Mme JANSSENS". */
|
|
925
|
+
const TITLE_BEFORE = /(?<![A-Za-z])(?:dr|prof|mr|mrs|ms|mme|mlle|m|herr|frau|dhr|mevr|sig|sra?)\.?\s+$/i;
|
|
926
|
+
const LETTER = /\p{L}/u;
|
|
927
|
+
/** A BIC/SWIFT label touching the token: "BIC: ", "SWIFT-Code:\n", "Code SWIFT : ", "BIC Bank A: ". */
|
|
928
|
+
const BIC_LABEL_BEFORE = /(?<![A-Za-z])(?:bic|swift)(?:[\s/\-]*(?:code|swift|bic))?(?:\s+bank\s+\w{1,3})?[\s:.\-()]*$/i;
|
|
929
|
+
/**
|
|
930
|
+
* A registry miss sitting where only a word or a name can (rules-engine#57):
|
|
931
|
+
* joined to a word by a hyphen ("NGUYEN-HOFFMANN"), right after a personal
|
|
932
|
+
* title ("Dr. HOFFMANN"), or letters only (eight, or eleven without an `XXX`
|
|
933
|
+
* branch: "BETALING", "MAANDELIJKS") that the GLEIF mapping does not know and
|
|
934
|
+
* no BIC/SWIFT label touches.
|
|
935
|
+
*/
|
|
936
|
+
function bicShapedWord(text, start, end, token) {
|
|
937
|
+
if ((start >= 2 && text[start - 1] === "-" && LETTER.test(text[start - 2])) ||
|
|
938
|
+
(text[end] === "-" && end + 1 < text.length && LETTER.test(text[end + 1])))
|
|
939
|
+
return true;
|
|
940
|
+
if (TITLE_BEFORE.test(text.slice(Math.max(0, start - 8), start)))
|
|
941
|
+
return true;
|
|
942
|
+
return /^[A-Za-z]+$/.test(token) && !(token.length === 11 && token.endsWith("XXX"))
|
|
943
|
+
&& !(0, bicRegistry_js_1.isKnownInstitution)(token)
|
|
944
|
+
&& !BIC_LABEL_BEFORE.test(text.slice(Math.max(0, start - 30), start));
|
|
945
|
+
}
|
|
734
946
|
// ── Postal code: digits belonging to a longer identifier ────────────────
|
|
735
947
|
// An identifier label immediately before the digits. A postal code is never
|
|
736
948
|
// introduced this way; an SVNr, policy number or service number always is.
|
|
@@ -738,12 +950,26 @@ function suppressBicWithoutEvidence(text, match, scratch) {
|
|
|
738
950
|
// Repeated per alternative, the engine retried the same unbounded prefix five
|
|
739
951
|
// times per starting position — the most expensive single regex in the engine
|
|
740
952
|
// at 197 ms per 1 MB document. Same language, 8x faster on the worst case.
|
|
953
|
+
// An identifier cue that is really a postal label -- see the use below.
|
|
954
|
+
const POSTAL_LABEL_CUE = /^post/i;
|
|
741
955
|
const ID_CUE_BEFORE = /(?:[\w\-]*(?:Nr|N[°ºo]|Nummer|Numero|Numéro)|No|number|num|Kennzahl|Aktenzeichen|Az|e-?card|Polizze|Police|Policen)\.?\s*:?\s*$/i;
|
|
742
956
|
// An international dialling prefix earlier on the same line, with nothing but
|
|
743
957
|
// number punctuation in between: these digits belong to the phone detector.
|
|
744
958
|
const DIALLING_PREFIX_BEFORE = /\+\d{1,3}[\d\s\-().]*$/;
|
|
745
959
|
// A country prefix on a postal code — "A-1010 Wien", "B-2000", "L-1234".
|
|
746
|
-
|
|
960
|
+
// The boundary excludes a hyphen, which is what makes this a *prefix* rather
|
|
961
|
+
// than a segment. It did not, so any `XX-NNNNN` tail of a hyphenated reference
|
|
962
|
+
// read as a country-prefixed address: in "PV-2026-LU-09143" the `LU` is
|
|
963
|
+
// preceded by `-` and the German five-digit postal pattern kept `09143`
|
|
964
|
+
// (rules-engine#31).
|
|
965
|
+
const COUNTRY_PREFIXED = /(?:^|[^A-Za-z0-9\-])([A-Z]{1,3})$/;
|
|
966
|
+
// Postal country prefixes and the length of that country's postcode: any one or two capitals
|
|
967
|
+
// used to pass, so "K-40817" and "B-51823" were country-prefixed postcodes (rules-engine#111).
|
|
968
|
+
const POSTAL_PREFIX_DIGITS = { A: 4, AT: 4, B: 4, BE: 4, BG: 4, CH: 4, CY: 4, CZ: 5, D: 5, DE: 5, DK: 4, E: 5, EE: 5, ES: 5, F: 5, FI: 5, FIN: 5, FR: 5, GR: 5, H: 4, HR: 5, I: 5, IT: 5, L: 4, LT: 5, LU: 4, LV: 4, N: 4, NL: 4, NO: 4, P: 4, PL: 5, PT: 4, RO: 6, S: 5, SE: 5, SI: 4, SK: 5, SLO: 4 };
|
|
969
|
+
function countryPrefixed(before, digits) {
|
|
970
|
+
const m = COUNTRY_PREFIXED.exec(before);
|
|
971
|
+
return m !== null && POSTAL_PREFIX_DIGITS[m[1]] === digits.length;
|
|
972
|
+
}
|
|
747
973
|
/**
|
|
748
974
|
* Suppress bare digit runs that belong to a longer number, not an address.
|
|
749
975
|
*
|
|
@@ -759,8 +985,15 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
759
985
|
if (match.patternDef.entityType !== types_js_1.EntityType.POSTAL_CODE)
|
|
760
986
|
return false;
|
|
761
987
|
const clean = match.text.trim();
|
|
762
|
-
if (!/^\d+$/.test(clean))
|
|
988
|
+
if (!/^\d+$/.test(clean)) {
|
|
989
|
+
// A structured form glued on by a hyphen is the tail of a reference: "CPS-HR-2026-019" (#111).
|
|
990
|
+
const glued = text.slice(Math.max(0, match.start - 2), match.start);
|
|
991
|
+
if (glued.length === 2 && glued[1] === "-" && /[A-Za-z0-9]/.test(glued[0])) {
|
|
992
|
+
const m = COUNTRY_PREFIXED.exec(text.slice(0, match.start - 1));
|
|
993
|
+
return !(m && m[1] in POSTAL_PREFIX_DIGITS);
|
|
994
|
+
}
|
|
763
995
|
return false;
|
|
996
|
+
}
|
|
764
997
|
const beforeChar = match.start > 0 ? text[match.start - 1] : "";
|
|
765
998
|
const afterChar = match.end < text.length ? text[match.end] : "";
|
|
766
999
|
const prevPrev = match.start >= 2 ? text[match.start - 2] : "";
|
|
@@ -779,7 +1012,10 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
779
1012
|
return true;
|
|
780
1013
|
if (afterChar === "-" && /\d/.test(nextNext))
|
|
781
1014
|
return true;
|
|
782
|
-
|
|
1015
|
+
// A hyphenated chain goes on after it: "réf. : L-2025-HD-0412" is a reference (#111).
|
|
1016
|
+
if (/^-[A-Za-z0-9]+-/.test(text.slice(match.end, match.end + 12)))
|
|
1017
|
+
return true;
|
|
1018
|
+
if (beforeChar === "-" && !countryPrefixed(text.slice(0, match.start - 1), clean))
|
|
783
1019
|
return true;
|
|
784
1020
|
// A further digit group on the same line: "1268 040390", "1234 5678 925",
|
|
785
1021
|
// "+43 664 8213 907". Horizontal whitespace only — a digit on the *next*
|
|
@@ -787,7 +1023,16 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
787
1023
|
if (/^[ \t]+\d/.test(text.slice(match.end, match.end + 4)))
|
|
788
1024
|
return true;
|
|
789
1025
|
// An identifier label introduces the digits: "DiNr. 4471", "Policen-Nr."
|
|
790
|
-
|
|
1026
|
+
// Unless the label is a *postal* one. ID_CUE_BEFORE matches any word ending
|
|
1027
|
+
// in "Nummer", "Nr", "Numero" or "Numéro" -- the wildcard is `[\w\-]*` -- so
|
|
1028
|
+
// it matches `postnummer`, `postnr` and `postinumero`, the canonical postal
|
|
1029
|
+
// labels of Norway, Denmark and Finland. Those countries write a bare
|
|
1030
|
+
// four- or five-digit code, so the isdigit() guard above lets this run and
|
|
1031
|
+
// the code was suppressed by its own label: `postnummer: 5020 Bergen`
|
|
1032
|
+
// produced nothing at all (rules-engine#41).
|
|
1033
|
+
const before = text.slice(Math.max(0, match.start - 40), match.start);
|
|
1034
|
+
const cue = ID_CUE_BEFORE.exec(before);
|
|
1035
|
+
if (cue !== null && !POSTAL_LABEL_CUE.test(cue[0].trimStart()))
|
|
791
1036
|
return true;
|
|
792
1037
|
// Digits after an international dialling prefix belong to PHONE
|
|
793
1038
|
const [lineStart] = enclosingLine(text, match.start, match.end);
|
|
@@ -834,12 +1079,158 @@ function suppressRequiresContext(text, match) {
|
|
|
834
1079
|
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
835
1080
|
fillsADelimitedField(text, match.start, match.end))
|
|
836
1081
|
return false;
|
|
1082
|
+
if (DATE_LABEL_TYPES.has(match.patternDef.entityType)) {
|
|
1083
|
+
return !labelsThisDate(text, match.start, match.end, match.patternDef.contextKeywords, match.patternDef.entityType);
|
|
1084
|
+
}
|
|
837
1085
|
const [before, after] = getContext(text, match.start, match.end);
|
|
838
1086
|
const context = (before + " " + after).toLowerCase();
|
|
839
1087
|
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
840
1088
|
}
|
|
1089
|
+
/** Date types decided by a label: DOB and DATE_OF_DEATH share one date shape. */
|
|
1090
|
+
const DATE_LABEL_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
|
|
1091
|
+
/** Any common numeric date, to tell whether a label belongs to another date. */
|
|
1092
|
+
// A masked date counts as one: the gateway's rules pass reads SDK-masked text (rules-engine#111).
|
|
1093
|
+
// ... and a written-month date, now a DOB shape of its own (rules-engine#115).
|
|
1094
|
+
const ANY_DATE = new RegExp(String.raw `\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}|\[(?:dob|date_of_death)\]|(?<![A-Za-z0-9_])\d{1,2}\.?\s+(?:` + exports.MONTH_NAMES + String.raw `)\.?\s+\d{4}(?![A-Za-z0-9_])`, "i");
|
|
1095
|
+
/**
|
|
1096
|
+
* True when one of `keywords` is this date's own label: before the date with no
|
|
1097
|
+
* other date between, or after it inside the date's own sentence. Anywhere in
|
|
1098
|
+
* the window typed every date from whichever date label the document carried
|
|
1099
|
+
* (rules-engine#52).
|
|
1100
|
+
*/
|
|
1101
|
+
/** A date followed by a list separator: an item of the list a label introduces. */
|
|
1102
|
+
const LIST_ITEM = /\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}\s*(?:,|;|–|&|\band\b|\ben\b|\bet\b|\bund\b|\bor\b|\bof\b|\bou\b|\boder\b|\brespectievelijk\b|\bbzw\.?)\s*/g;
|
|
1103
|
+
/** Punctuation and at most two short words ("e.g.", "bv.", "op") between label and date. */
|
|
1104
|
+
const LABEL_TAIL = /^[\s:.,()\-]*(?:\p{L}{1,4}(?:\.\p{L}{1,2})*\.?(?:[\s:.,()\-]+|$)){0,2}$/u;
|
|
1105
|
+
/** The birth sign "°" directly before the date, but not the number sign "n°". */
|
|
1106
|
+
const BIRTH_SIGN = /(?<![a-z])°\s?$/;
|
|
1107
|
+
/** No other date between a label and the date, except items of its list. */
|
|
1108
|
+
function nothingButAListBetween(gap) {
|
|
1109
|
+
if (!ANY_DATE.test(gap))
|
|
1110
|
+
return true;
|
|
1111
|
+
const rest = gap.replace(LIST_ITEM, " ");
|
|
1112
|
+
return !ANY_DATE.test(rest) && LABEL_TAIL.test(rest);
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Where `keyword` occurs in the lower-cased `haystack`, or -1. An all-capitals
|
|
1116
|
+
* keyword is an abbreviation and counts only as a word of its own: "DOB" is a
|
|
1117
|
+
* label, the "dob" in Polish "dobrowolnej" is not (rules-engine#100); a plural
|
|
1118
|
+
* "s" may follow ("DOBs e.g. …"). Other
|
|
1119
|
+
* keywords still match inside a longer word, which inflected forms need.
|
|
1120
|
+
*/
|
|
1121
|
+
function keywordAt(haystack, keyword, last) {
|
|
1122
|
+
const kw = keyword.toLowerCase();
|
|
1123
|
+
if (keyword !== keyword.toUpperCase() || keyword === kw) {
|
|
1124
|
+
return last ? haystack.lastIndexOf(kw) : haystack.indexOf(kw);
|
|
1125
|
+
}
|
|
1126
|
+
const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
|
|
1127
|
+
const hits = [];
|
|
1128
|
+
for (let i = haystack.indexOf(kw); i !== -1; i = haystack.indexOf(kw, i + 1)) {
|
|
1129
|
+
let next = i + kw.length;
|
|
1130
|
+
if (haystack[i - 1] === "[" && haystack[next] === "]")
|
|
1131
|
+
continue; // a placeholder ("[DOB]"), not a label (#111)
|
|
1132
|
+
if (haystack[next] === "s")
|
|
1133
|
+
next++;
|
|
1134
|
+
if (!isLetter(haystack.slice(i - 1, i)) && !isLetter(haystack.slice(next, next + 1))) {
|
|
1135
|
+
hits.push(i);
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
if (hits.length === 0)
|
|
1139
|
+
return -1;
|
|
1140
|
+
return last ? hits[hits.length - 1] : hits[0];
|
|
1141
|
+
}
|
|
1142
|
+
function labelsThisDate(text, start, end, keywords, entityType) {
|
|
1143
|
+
const [before, after] = getContext(text, start, end);
|
|
1144
|
+
const lowerBefore = before.toLowerCase();
|
|
1145
|
+
const lowerAfter = after.toLowerCase();
|
|
1146
|
+
// A label on an earlier line counts only when the date opens its own line or
|
|
1147
|
+
// the label was asked as a question (a call transcript's answer turn);
|
|
1148
|
+
// a column header is a fallback and never overrules a label of the date's own.
|
|
1149
|
+
const lineStart = lowerBefore.lastIndexOf("\n") + 1;
|
|
1150
|
+
const opensLine = lowerBefore.slice(lineStart).replace(/[ \t:|;*\-•(\[]/g, "") === "";
|
|
1151
|
+
if (entityType === types_js_1.EntityType.DOB && BIRTH_SIGN.test(lowerBefore))
|
|
1152
|
+
return true;
|
|
1153
|
+
for (const keyword of keywords) {
|
|
1154
|
+
const kw = keyword.toLowerCase();
|
|
1155
|
+
const i = keywordAt(lowerBefore, keyword, true);
|
|
1156
|
+
if (i >= 0 && nothingButAListBetween(lowerBefore.slice(i + kw.length))
|
|
1157
|
+
&& (i >= lineStart || opensLine
|
|
1158
|
+
|| lowerBefore.slice(i + kw.length, lineStart).includes("?")))
|
|
1159
|
+
return true;
|
|
1160
|
+
const j = keywordAt(lowerAfter, keyword, false);
|
|
1161
|
+
if (j >= 0) {
|
|
1162
|
+
const gap = lowerAfter.slice(0, j);
|
|
1163
|
+
if (!ANY_DATE.test(gap) && !SENTENCE_BREAK.test(gap)
|
|
1164
|
+
&& !LEADS_TO_A_DATE.test(lowerAfter.slice(j + kw.length)))
|
|
1165
|
+
return true;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
const header = columnHeader(text, start);
|
|
1169
|
+
return header !== null && keywords.some(kw => keywordAt(header, kw, false) >= 0);
|
|
1170
|
+
}
|
|
1171
|
+
/**
|
|
1172
|
+
* A label running straight into a date of its own introduces that date, not the
|
|
1173
|
+
* one before it ("Verstorben am 01.02.2020, geboren am 12.03.1940").
|
|
1174
|
+
*/
|
|
1175
|
+
// ... or into a value already masked ("né le [DOB]"): the label is that value's (rules-engine#111).
|
|
1176
|
+
const LEADS_TO_A_DATE = new RegExp(String.raw `^[\s:.\-]*(?:[a-zà-ÿ]{1,3}[\s:.]+)?(?:\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}|\[[a-z_]+\]|\d{1,2}\.?\s+(?:` + exports.MONTH_NAMES + String.raw `)\.?\s+\d{4})`, "i");
|
|
1177
|
+
/** Field separators a table row may use. */
|
|
1178
|
+
const COLUMN_SEPARATORS = [";", "|", "\t", ","];
|
|
1179
|
+
/**
|
|
1180
|
+
* The lower-cased header cell above the value, if it sits in a table: in a table
|
|
1181
|
+
* the column decides the label, not proximity. A header is one of the three
|
|
1182
|
+
* lines above, split by the row's separator into the same number of cells.
|
|
1183
|
+
*/
|
|
1184
|
+
function columnHeader(text, start) {
|
|
1185
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
1186
|
+
const lineEndAt = text.indexOf("\n", start);
|
|
1187
|
+
const line = text.slice(lineStart, lineEndAt >= 0 ? lineEndAt : text.length);
|
|
1188
|
+
// The three lines above, by lastIndexOf: splitting the whole prefix was
|
|
1189
|
+
// quadratic in the document, once per date candidate.
|
|
1190
|
+
let aboveStart = lineStart - 1;
|
|
1191
|
+
for (let k = 0; k < 3 && aboveStart >= 0; k++) {
|
|
1192
|
+
aboveStart = aboveStart === 0 ? -1 : text.lastIndexOf("\n", aboveStart - 1);
|
|
1193
|
+
}
|
|
1194
|
+
const above = lineStart ? text.slice(aboveStart + 1, Math.max(0, lineStart - 1)).split("\n") : [];
|
|
1195
|
+
for (const sep of COLUMN_SEPARATORS) {
|
|
1196
|
+
if (!line.includes(sep))
|
|
1197
|
+
continue;
|
|
1198
|
+
const cells = line.split(sep);
|
|
1199
|
+
const column = text.slice(lineStart, start).split(sep).length - 1;
|
|
1200
|
+
for (const header of [...above].reverse()) {
|
|
1201
|
+
// A "label : value" field with a comma in its value is not a header (#111).
|
|
1202
|
+
if (header.includes(":"))
|
|
1203
|
+
continue;
|
|
1204
|
+
const heads = header.split(sep);
|
|
1205
|
+
if (heads.length === cells.length && !ANY_DATE.test(header))
|
|
1206
|
+
return heads[column].toLowerCase();
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
return null;
|
|
1210
|
+
}
|
|
1211
|
+
/** A dotted MAC shape glued to more of a reference is part of it: the tail of
|
|
1212
|
+
* "WSO-II.6151.4471.2025" (rules-engine#105). Dotted form only. */
|
|
1213
|
+
function suppressMacInReference(text, match) {
|
|
1214
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.MAC_ADDRESS || !match.text.includes("."))
|
|
1215
|
+
return false;
|
|
1216
|
+
return joinedToMore(text, match.start, match.end);
|
|
1217
|
+
}
|
|
1218
|
+
/** A card number is never carved out of a longer run of digit groups: a
|
|
1219
|
+
* four-digit group joined by the card's own separator on either side means the
|
|
1220
|
+
* run is longer, e.g. a Polish account number (rules-engine#106). */
|
|
1221
|
+
function suppressCardInsideDigitRun(text, match) {
|
|
1222
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.CREDIT_CARD)
|
|
1223
|
+
return false;
|
|
1224
|
+
const sep = [...match.text].find(c => !/\d/.test(c)) ?? "";
|
|
1225
|
+
if (!sep)
|
|
1226
|
+
return false;
|
|
1227
|
+
const s = sep.replace(/[-\\^$.*+?()[\]{}|]/g, "\\$&");
|
|
1228
|
+
const before = text.slice(Math.max(0, match.start - 6), match.start);
|
|
1229
|
+
const after = text.slice(match.end, match.end + 6);
|
|
1230
|
+
return new RegExp(`(?<!\\d)\\d{4}${s}$`).test(before) || new RegExp(`^${s}\\d{4}(?!\\d)`).test(after);
|
|
1231
|
+
}
|
|
841
1232
|
const TYPE_SUPPRESSORS = {
|
|
842
|
-
[types_js_1.EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneServiceNumber, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
1233
|
+
[types_js_1.EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneServiceNumber, suppressPhoneInsideAccountRun, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
843
1234
|
[types_js_1.EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
844
1235
|
[types_js_1.EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
845
1236
|
[types_js_1.EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
@@ -849,10 +1240,14 @@ const TYPE_SUPPRESSORS = {
|
|
|
849
1240
|
[types_js_1.EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
850
1241
|
[types_js_1.EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
851
1242
|
[types_js_1.EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
1243
|
+
[types_js_1.EntityType.MAC_ADDRESS]: [suppressMacInReference],
|
|
1244
|
+
[types_js_1.EntityType.CREDIT_CARD]: [suppressCardInsideDigitRun],
|
|
852
1245
|
};
|
|
853
1246
|
function shouldSuppress(text, match, scratch) {
|
|
854
1247
|
if (suppressSequential(text, match))
|
|
855
1248
|
return true;
|
|
1249
|
+
if (suppressRedactionPlaceholder(text, match))
|
|
1250
|
+
return true;
|
|
856
1251
|
const typeSups = TYPE_SUPPRESSORS[match.patternDef.entityType];
|
|
857
1252
|
if (typeSups) {
|
|
858
1253
|
for (const s of typeSups) {
|