euredact 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -194,7 +194,7 @@ const BE = {
|
|
|
194
194
|
p(types_js_1.EntityType.NATIONAL_ID, String.raw `\b\d{2}\.?\d{2}\.?\d{2}[\-.]?\d{3}\.?\d{2}\b`, "belgian_nn"),
|
|
195
195
|
p(types_js_1.EntityType.IBAN, String.raw `\bBE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\b`, "iban"),
|
|
196
196
|
p(types_js_1.EntityType.VAT, String.raw `\bBE\s?0\d{3}\.?\d{3}\.?\d{3}\b`, "belgian_vat"),
|
|
197
|
-
p(types_js_1.EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer"], true),
|
|
197
|
+
p(types_js_1.EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer", "Kruispuntbank", "Banque-Carrefour", "Banque Carrefour", "Ondernemingen", "Entreprises", "onder nummer"], true),
|
|
198
198
|
p(types_js_1.EntityType.PHONE, String.raw `\b0[1-9]\d{0,2}[/\s.\-]?\d{2,3}[.\s\-]?\d{2,3}[.\s\-]?\d{2,3}\b`),
|
|
199
199
|
p(types_js_1.EntityType.PHONE, String.raw `\+32\s?\d{1,3}[\s.\-]?\d{2,3}[\s.\-]?\d{2}[\s.\-]?\d{2}`),
|
|
200
200
|
p(types_js_1.EntityType.PASSPORT, String.raw `\b[A-Z]{2}\d{6}\b`, null, "", ["paspoort", "passport", "passeport", "reisdocument", "travel document", "document de voyage"], true),
|
|
@@ -387,8 +387,15 @@ const ES = {
|
|
|
387
387
|
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}\b`),
|
|
388
388
|
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d{8}\b`),
|
|
389
389
|
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
390
|
+
// 3-2-2-2. Just as common as the groupings above and absent until now:
|
|
391
|
+
// "705 97 55 11" matched no phone pattern at all, which cost 674 Spanish
|
|
392
|
+
// numbers — the single largest PHONE gap.
|
|
393
|
+
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
390
394
|
p(types_js_1.EntityType.PHONE, String.raw `\+34\s?[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}`),
|
|
391
|
-
|
|
395
|
+
// A single space, not \s: \s matched a line break, so a year ending one
|
|
396
|
+
// line and a label starting the next ("2002\nCPR") read as one
|
|
397
|
+
// registration — 310 times across the corpus.
|
|
398
|
+
p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b\d{4}[ ]?[BCDFGHJKLMNPRSTVWXYZ]{3}\b`),
|
|
392
399
|
p(types_js_1.EntityType.POSTAL_CODE, String.raw `\b(?:0[1-9]|[1-4]\d|5[0-2])\d{3}\b`, null, "", ["código postal", "C.P.", "CP", "dirección", "domicilio", "calle", "avenida", "plaza", "paseo", "camino", "Postal:"], true),
|
|
393
400
|
],
|
|
394
401
|
};
|
package/dist/cjs/rules/engine.js
CHANGED
|
@@ -90,6 +90,46 @@ function warnUnknownCountry(code) {
|
|
|
90
90
|
`shared country-independent patterns only (email, IBAN, international ` +
|
|
91
91
|
`phone, credit card, ...). Available: ${available.join(", ")}`);
|
|
92
92
|
}
|
|
93
|
+
// ── Local cues: what the document calls the value next to it ────────────
|
|
94
|
+
//
|
|
95
|
+
// Country evidence resolves which *national scheme* owns an ambiguous value, but
|
|
96
|
+
// it never looks at the label sitting immediately in front of it. So
|
|
97
|
+
// "Phone: 0705237535" was reported as a Swedish national ID: the digits pass
|
|
98
|
+
// that checksum, the document is Swedish, and nothing consulted the word
|
|
99
|
+
// "Phone".
|
|
100
|
+
//
|
|
101
|
+
// A cue touching the span is the strongest statement the document makes about
|
|
102
|
+
// what the value *is*, so it outranks the checksum — but only among candidates
|
|
103
|
+
// covering the **same span**, so it decides the label and can never change which
|
|
104
|
+
// characters are masked. That is what keeps invariant I1 intact.
|
|
105
|
+
//
|
|
106
|
+
// Measured: 263 phone numbers and 18 French SIREN were reported as national IDs,
|
|
107
|
+
// each with an explicit cue in front of it.
|
|
108
|
+
const LOCAL_CUES = [
|
|
109
|
+
[types_js_1.EntityType.PHONE,
|
|
110
|
+
/(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell)\w*\s*[:.\-]?\s*$/i],
|
|
111
|
+
[types_js_1.EntityType.CHAMBER_OF_COMMERCE,
|
|
112
|
+
/(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr)\w*\s*[:.\-]?\s*$/i],
|
|
113
|
+
[types_js_1.EntityType.VAT,
|
|
114
|
+
/(?:vat|btw|tva|ust|iva|moms|alv|mwst)[\w.\-]*\s*[:.\-]?\s*$/i],
|
|
115
|
+
[types_js_1.EntityType.BIC,
|
|
116
|
+
/(?:bic|swift)[\w/]*\s*[:.\-]?\s*\(?\s*$/i],
|
|
117
|
+
[types_js_1.EntityType.NATIONAL_ID,
|
|
118
|
+
/(?:bsn|personnummer|rijksregisternummer|national\s*id|identiteitsnummer|henkilötunnus|kennitala|cpr|nif|dni)\w*\s*[:.\-]?\s*$/i],
|
|
119
|
+
];
|
|
120
|
+
// How far back a cue may sit. Short on purpose: a cue is only evidence about the
|
|
121
|
+
// value it introduces, which is the lesson of the postal-code year defect, where
|
|
122
|
+
// a cue 150 characters away licensed every number in the window.
|
|
123
|
+
const CUE_WINDOW = 22;
|
|
124
|
+
/** 1 when a cue naming `entityType` sits immediately before the span. */
|
|
125
|
+
function localCueBonus(text, start, entityType) {
|
|
126
|
+
const before = text.slice(Math.max(0, start - CUE_WINDOW), start);
|
|
127
|
+
for (const [cuedType, pattern] of LOCAL_CUES) {
|
|
128
|
+
if (cuedType === entityType && pattern.test(before))
|
|
129
|
+
return 1;
|
|
130
|
+
}
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
93
133
|
class RuleEngine {
|
|
94
134
|
matcher = new matchers_js_1.MultiPatternMatcher();
|
|
95
135
|
loadedCountries = new Set();
|
|
@@ -373,6 +413,7 @@ class RuleEngine {
|
|
|
373
413
|
? match.countryCode : null;
|
|
374
414
|
candidates.push({
|
|
375
415
|
blindPriority,
|
|
416
|
+
cue: localCueBonus(text, match.start, match.patternDef.entityType),
|
|
376
417
|
priority,
|
|
377
418
|
length: match.end - match.start,
|
|
378
419
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
@@ -390,7 +431,7 @@ class RuleEngine {
|
|
|
390
431
|
});
|
|
391
432
|
}
|
|
392
433
|
for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
|
|
393
|
-
candidates.push({ blindPriority: 1, priority: 1, length: d.end - d.start,
|
|
434
|
+
candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
|
|
394
435
|
countryScore: 0, inScope: 1, det: d });
|
|
395
436
|
}
|
|
396
437
|
const detections = this.deduplicate(candidates);
|
|
@@ -453,9 +494,10 @@ class RuleEngine {
|
|
|
453
494
|
const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first } country-blind
|
|
454
495
|
tierOf(b) - tierOf(a) || // best tier here } — decides
|
|
455
496
|
a.det.start - b.det.start || // leftmost } WHICH span
|
|
456
|
-
b.
|
|
457
|
-
b.
|
|
458
|
-
b.
|
|
497
|
+
b.cue - a.cue || // local cue } country-aware
|
|
498
|
+
b.priority - a.priority || // country priority } — decides the
|
|
499
|
+
b.countryScore - a.countryScore || // country evidence } LABEL only
|
|
500
|
+
b.inScope - a.inScope); // declared scope }
|
|
459
501
|
const result = [];
|
|
460
502
|
const occupied = new Set();
|
|
461
503
|
for (const { det } of sorted) {
|
|
@@ -12,14 +12,29 @@ function getContext(text, start, end) {
|
|
|
12
12
|
}
|
|
13
13
|
// Unicode-safe word boundary: negative lookahead for any letter (including accented)
|
|
14
14
|
const _UWB = "(?![a-zA-Z\\u00C0-\\u024F\\u0400-\\u04FF])";
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
// Symbols are held apart from the alphabetic codes because a word boundary
|
|
16
|
+
// after "€" is meaningless — the symbol is not a word character. Python used a
|
|
17
|
+
// bare "\b" here and so could never match "1163 €," at all; this port used the
|
|
18
|
+
// lookahead below and did not have that defect. Kept split so the difference
|
|
19
|
+
// cannot come back.
|
|
20
|
+
const CURRENCY_SYMBOL = "€|\\$|£|¥|₺|zł|Kč|лв|kn|Ft|₽";
|
|
21
|
+
const CURRENCY_WORD = "EUR|USD|GBP|CHF|ISK|SEK|NOK|DKK|PLN|CZK|HUF|RON|BGN|HRK|" +
|
|
22
|
+
"euro|euros|dollar|dollars|pond|kronor|kroner|kronur|kr|" +
|
|
23
|
+
"złoty|korun|forint|lei|leva";
|
|
24
|
+
const CURRENCY_AFTER = new RegExp(`^\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
25
|
+
const CURRENCY_COMMA_AFTER = new RegExp(`^[.,]\\d{1,2}\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
26
|
+
const CURRENCY_BEFORE = new RegExp(`(?:(?:${CURRENCY_SYMBOL})|\\b(?:${CURRENCY_WORD}))\\s*$`, "i");
|
|
27
|
+
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)\s*(?:\w+\s*)?:?\s*$/i;
|
|
19
28
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
20
29
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
21
30
|
const UNIT_AFTER = /^\s*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
22
|
-
|
|
31
|
+
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
32
|
+
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|ticket|incident\s*(?:report|nr|no)?|case\s*(?:nr|no|id)?|zaaknummer|meldingsnummer|Vorgangsnummer|Vorgang|Störungsmeldung|saksnummer|ärendenummer|sagsnummer|asianumero|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.#]?\s*$/i;
|
|
33
|
+
// "#" before a number marks a reference in every language in scope — a ticket,
|
|
34
|
+
// an order, a line item — and never a postcode or a national identifier.
|
|
35
|
+
const HASH_BEFORE = /#\s*$/;
|
|
36
|
+
// A short uppercase tag hyphenated to the number: "IR-43433", "INC-2024".
|
|
37
|
+
const REF_PREFIX_BEFORE = /(?:^|[\s([])[A-Z]{2,5}-$/;
|
|
23
38
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
24
39
|
const MATH_BEFORE = /[=+\-×÷*/]\s*$/;
|
|
25
40
|
const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
@@ -28,12 +43,204 @@ const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
|
28
43
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
29
44
|
const SEQUENTIAL_PATTERNS = /^(?:0{6,}|1234567890?|0123456789|9876543210?|1111111111?|000000000|123456789)$/;
|
|
30
45
|
// YEAR_PATTERN kept for reference but replaced by RECENT_YEAR in suppressYearAsPostal
|
|
31
|
-
|
|
46
|
+
// The leading \b is load-bearing. Without it the short alternatives match the
|
|
47
|
+
// *tail* of an ordinary word: "NIS" matched "tālru**nis** ", the Latvian for
|
|
48
|
+
// "telephone", so every Latvian phone number was suppressed as though it sat
|
|
49
|
+
// behind a Belgian national-ID label. 653 misses from a missing word boundary.
|
|
50
|
+
const ID_LABEL_BEFORE = /\b(?:BSN|RR|NN|NIR|INSZ|NISS|NIS|Steuer-?ID|TIN|NIF|NIE|SSN|rijksregisternummer|numéro\s*national|national\s*number|matricule|Ausweisnummer|Personalausweis|Versichertennummer|KVNR|KV-Nr|Steuernummer|St\.\-Nr|StNr|Finanzamt\s+ist|Ondernemingen\s+onder\s+nummer|ondernemingsnummer|numéro\s*d'entreprise|enterprise\s*number|Kruispuntbank)\s*[:.]?\s*$/i;
|
|
32
51
|
const SERVICE_NUMBER = /^0800[\-\s]/;
|
|
33
52
|
const DATE_PATTERN_FULL = /^\d{2}[-/.]\d{2}[-/.]\d{4}$|^\d{4}[-/.]\d{2}[-/.]\d{2}$/;
|
|
34
53
|
const PASSPORT_CONTEXT_BEFORE = /(?:Reisepass|passport|passeport|paspoort|Bisheriger\s+Reisepass)\s*(?:Nr\.?|Nummer|nummer|number|n[°o])?\s*[:.]?\s*$/i;
|
|
35
54
|
const SE_ORG_CONTEXT_BEFORE = /(?:org\.?\s*nr\.?|organisationsnummer|organisationsnr|Bolagsverket|företag)\s*[:.]?\s*$/i;
|
|
36
55
|
const NUMERIC_TYPES = new Set([types_js_1.EntityType.PHONE, types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID, types_js_1.EntityType.POSTAL_CODE]);
|
|
56
|
+
// ── Date adjacency, for the postal-code year gate ───────────────────────
|
|
57
|
+
//
|
|
58
|
+
// Month names in the languages this engine covers. Long, but it is data: the
|
|
59
|
+
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
60
|
+
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
61
|
+
const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
62
|
+
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
63
|
+
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
64
|
+
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
65
|
+
"|januari|februari|maart|mei|augustus|mrt" +
|
|
66
|
+
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
67
|
+
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
68
|
+
"|janeiro|fevereiro|março|marco|maio|junho|julho|setembro|outubro|novembro|dezembro" +
|
|
69
|
+
// "desember" (nb/nn/is) was the one December spelling missing from a list
|
|
70
|
+
// that already carried ten others — "1. desember 2025" was a Norwegian
|
|
71
|
+
// postcode because of it.
|
|
72
|
+
"|marts|augusti|desember" +
|
|
73
|
+
"|janúar|febrúar|apríl|maí|júní|júlí|ágúst|nóvember" +
|
|
74
|
+
"|tammikuu|helmikuu|maaliskuu|huhtikuu|toukokuu|kesäkuu|heinäkuu|elokuu|syyskuu|lokakuu|marraskuu|joulukuu" +
|
|
75
|
+
"|jaanuar|veebruar|aprill|juuni|juuli|oktoober|detsember" +
|
|
76
|
+
"|janvāris|februāris|aprīlis|maijs|jūnijs|jūlijs|augusts|septembris|oktobris|novembris|decembris" +
|
|
77
|
+
"|stycznia|lutego|marca|kwietnia|maja|czerwca|lipca|sierpnia|września|października|listopada|grudnia" +
|
|
78
|
+
"|ledna|února|března|dubna|května|června|července|srpna|září|října|listopadu|prosince" +
|
|
79
|
+
"|marec|junij|julij|avgust|siječnja|veljače|ožujka|travnja|svibnja|lipnja|srpnja|kolovoza|rujna|studenoga|prosinca" +
|
|
80
|
+
"|március|április|május|június|július|augusztus|szeptember|október" +
|
|
81
|
+
"|ianuarie|februarie|martie|aprilie|iunie|iulie|septembrie|octombrie|noiembrie|decembrie" +
|
|
82
|
+
"|ιανουαρίου|φεβρουαρίου|μαρτίου|απριλίου|μαΐου|ιουνίου|ιουλίου|αυγούστου|σεπτεμβρίου|οκτωβρίου|νοεμβρίου|δεκεμβρίου" +
|
|
83
|
+
"|януари|февруари|март|април|май|юни|юли|август|септември|октомври|ноември|декември";
|
|
84
|
+
/** "since" cues, which introduce a bare year in every one of these languages. */
|
|
85
|
+
const SINCE_CUES = "seit|since|depuis|sinds|sedert|vanaf|desde|dal|od|sedan|siden|alates|alkaen|από|от|din|iz|ab";
|
|
86
|
+
/**
|
|
87
|
+
* A month name, a "since" cue, or a numeric date tail immediately before the
|
|
88
|
+
* candidate. Adjacent-only on purpose — a cue 150 characters away is what made
|
|
89
|
+
* this a bug in the first place.
|
|
90
|
+
*/
|
|
91
|
+
const DATE_BEFORE = new RegExp(`(?:\\b(?:${MONTH_NAMES})\\b\\.?|\\b(?:${SINCE_CUES})\\b|\\d{1,2}[./-]\\d{1,2}[./-])\\s*$`, "iu");
|
|
92
|
+
/** A date separator immediately after, making the candidate a leading year. */
|
|
93
|
+
const DATE_AFTER = /^[./-]\d{1,2}[./-]\d{1,2}\b/;
|
|
94
|
+
/**
|
|
95
|
+
* ISO 4217 codes made only of the consonants a Spanish plate accepts, so a money
|
|
96
|
+
* amount reads as a registration. This is why "2297 DKK" was a plate.
|
|
97
|
+
*/
|
|
98
|
+
// Crypto tickers are the same shape and the same mistake — every surviving
|
|
99
|
+
// plate false positive was "4499 BTC" or a sibling. A ticker is a unit, exactly
|
|
100
|
+
// as an ISO 4217 code is.
|
|
101
|
+
const CURRENCY_CODE = "(?:DKK|SEK|NOK|ISK|CZK|PLN|HUF|RON|BGN|HRK|CHF|GBP|TRY|RSD|MKD|BYN|KZT|CNY|JPY|KRW|ZAR|BRL|MXN|CLP|COP|PLZ|SKK|TRL" +
|
|
102
|
+
"|BTC|ETH|LTC|XRP|BCH|XLM|XMR|DOT|SOL|ADA|TRX|DOGE|USDT|USDC)";
|
|
103
|
+
const CURRENCY_CODE_TAIL = new RegExp(`\\s${CURRENCY_CODE}$`);
|
|
104
|
+
const CURRENCY_CODE_AFTER = new RegExp(`^\\s*${CURRENCY_CODE}\\b`);
|
|
105
|
+
/** Log-timestamp tails, which the ":"-anchored secret rule picks up. */
|
|
106
|
+
const TIMESTAMP_FRAGMENT = /^\d{1,2}:\d{2}(?::\d{2})?(?:[.,]\d{1,6})?Z?$/;
|
|
107
|
+
/** Cloud region names and similar well-known configuration values. */
|
|
108
|
+
const NOT_A_SECRET = new RegExp("^(?:af|ap|ca|cn|eu|il|me|sa|us|gov)-(?:north|south|east|west|central" +
|
|
109
|
+
"|northeast|northwest|southeast|southwest)-\\d[a-z]?$" +
|
|
110
|
+
"|^(?:application|text|image|audio|video|multipart)/[\\w.+-]+$" +
|
|
111
|
+
"|^(?:utf|iso|windows)-[\\d-]+$" +
|
|
112
|
+
"|^(?:no-cache|no-store|max-age|gzip|deflate|identity|chunked)\\b", "i");
|
|
113
|
+
const EXACT_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
|
|
114
|
+
const BIC_SHAPE = /^[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?$/;
|
|
115
|
+
const BIC_CUE_BEFORE = /(?:BIC|SWIFT|BIC\/SWIFT)\s*[:=]?\s*\(?\s*$/i;
|
|
116
|
+
/** An email address, exactly. Same argument as the UUID and the BIC. */
|
|
117
|
+
const EXACT_EMAIL = /^[\w.!#$%&'*+/=?^`{|}~-]+@[\w-]+(?:\.[\w-]+)+$/;
|
|
118
|
+
/**
|
|
119
|
+
* A URL, or the tail of one after the scheme has been split off. The secret
|
|
120
|
+
* rules anchor on ":" and "=", so "https://api.sendgrid.com/v3/mail/send" hands
|
|
121
|
+
* them "//api.sendgrid.com/v3/mail/send" and a slash-separated path scores as
|
|
122
|
+
* high-entropy. An endpoint is published documentation, not a credential.
|
|
123
|
+
*/
|
|
124
|
+
const URL_LIKE = /^(?:[a-z][a-z0-9+.-]*:)?\/\/[^\s/]+(?:\/\S*)?$|^[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z]{2,}(?::\d{1,5})?(?:\/\S*)?$/i;
|
|
125
|
+
/**
|
|
126
|
+
* A URL that *carries* a credential is the opposite case, and it is the common
|
|
127
|
+
* one: "mongodb://admin:DFKDKi1eb51OOhuHPYz@rds-main.eu-west-1.rds.amazonaws.com"
|
|
128
|
+
* is a connection string with a live password in it. Suppressing those as "just
|
|
129
|
+
* a URL" lost 347 real secrets, so the endpoint test must not fire when userinfo
|
|
130
|
+
* or a credential-bearing query parameter is present.
|
|
131
|
+
*/
|
|
132
|
+
const URL_WITH_CREDENTIALS = /^(?:[a-z][a-z0-9+.-]*:)?\/\/[^/@\s]*:[^/@\s]*@|[?&](?:api_?key|access_?token|auth|token|secret|password|pwd|sig|signature|credential)=/i;
|
|
133
|
+
/** An LDAP distinguished name: "cn=github-actions,dc=corp,dc=eu". */
|
|
134
|
+
const LDAP_DN = /^(?:cn|ou|dc|uid|o|l|st|c)=[^,=]+(?:,\s*(?:cn|ou|dc|uid|o|l|st|c)=[^,=]+)+$/i;
|
|
135
|
+
/**
|
|
136
|
+
* Lower-case words joined by hyphens, optionally with a short id suffix:
|
|
137
|
+
* "service-account", "data-lake-33e061". The single-word test below already
|
|
138
|
+
* rejects "sozialversicherungsnummer" on the reasoning that a real secret would
|
|
139
|
+
* be mixed case; a hyphenated compound is the same token with a separator.
|
|
140
|
+
*/
|
|
141
|
+
const HYPHENATED_WORDS = /^[a-z]{2,}(?:-[a-z]{2,})+(?:-[0-9a-f]{4,8})?$/;
|
|
142
|
+
/** A dotted quad with every octet in range. */
|
|
143
|
+
const DOTTED_QUAD = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
|
|
144
|
+
/** Two plausible years joined by a hyphen — a school year, a contract term. */
|
|
145
|
+
const YEAR_RANGE = /^(19|20)\d{2}\s?[-–/]\s?(19|20)\d{2}$/;
|
|
146
|
+
/** A digit and a decimal point immediately before, so the match is a fraction. */
|
|
147
|
+
const DECIMAL_TAIL_BEFORE = /\d[.,]$/;
|
|
148
|
+
/**
|
|
149
|
+
* Punctuation the greedy "[^\s]{8,}" secret rule sweeps up with the token: it
|
|
150
|
+
* matched "TRIONL2U)." out of "(BIC: TRIONL2U).". Stripped only for the shape
|
|
151
|
+
* tests — the emitted span is never altered here.
|
|
152
|
+
*/
|
|
153
|
+
const PUNCT_EDGES = /^[\s([{<"'`]+|[\s)\]}>"'`.,;:!?]+$/g;
|
|
154
|
+
function coreToken(raw) {
|
|
155
|
+
return raw.replace(PUNCT_EDGES, "").replace(PUNCT_EDGES, "");
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* A generic secret must not claim a span that is a specific known type.
|
|
159
|
+
*
|
|
160
|
+
* The high-entropy rules are deliberately broad and they carry a validator, so
|
|
161
|
+
* they reach the top priority tier while the structured detector for the same
|
|
162
|
+
* characters sits at the bottom with nothing to offer. Measured on 152,300
|
|
163
|
+
* documents: 687 UUIDs and 140 BICs were reported as SECRET, each counted twice
|
|
164
|
+
* over — a false positive for SECRET and a miss for the type that should have
|
|
165
|
+
* had it.
|
|
166
|
+
*/
|
|
167
|
+
function suppressSecretOverStructured(text, match) {
|
|
168
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.SECRET)
|
|
169
|
+
return false;
|
|
170
|
+
const token = coreToken(match.text);
|
|
171
|
+
if (EXACT_UUID.test(token) || EXACT_EMAIL.test(token))
|
|
172
|
+
return true;
|
|
173
|
+
if (BIC_SHAPE.test(token)) {
|
|
174
|
+
return BIC_CUE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
|
|
175
|
+
}
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Reject the three things the ":"-anchored secret rule reliably mistakes:
|
|
180
|
+
* timestamps ("57:22.283Z]"), ordinary words ("Sozialversicherungsnummer") and
|
|
181
|
+
* region names ("us-east-1"). All three sit after a colon or equals sign, which
|
|
182
|
+
* is all that rule asks for, and all three clear the entropy threshold.
|
|
183
|
+
*
|
|
184
|
+
* The word test is narrow on purpose: purely alphabetic *and* shaped like one
|
|
185
|
+
* natural word. A random all-letter token would be mixed case and is left alone
|
|
186
|
+
* — losing a real secret is the expensive direction.
|
|
187
|
+
*/
|
|
188
|
+
function suppressSecretNotASecret(_text, match) {
|
|
189
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.SECRET)
|
|
190
|
+
return false;
|
|
191
|
+
const bare = coreToken(match.text);
|
|
192
|
+
if (TIMESTAMP_FRAGMENT.test(bare))
|
|
193
|
+
return true;
|
|
194
|
+
if (NOT_A_SECRET.test(bare))
|
|
195
|
+
return true;
|
|
196
|
+
if (URL_LIKE.test(bare) && !URL_WITH_CREDENTIALS.test(bare))
|
|
197
|
+
return true;
|
|
198
|
+
if (LDAP_DN.test(bare))
|
|
199
|
+
return true;
|
|
200
|
+
if (HYPHENATED_WORDS.test(bare))
|
|
201
|
+
return true;
|
|
202
|
+
const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
|
|
203
|
+
const oneWord = bare === bare.toLowerCase() ||
|
|
204
|
+
(bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
|
|
205
|
+
bare.slice(1) === bare.slice(1).toLowerCase());
|
|
206
|
+
return alpha && oneWord;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* A money amount followed by its ISO 4217 code is not a registration. Spain's
|
|
210
|
+
* plate shape is four digits then three consonants, and the Nordic and Central
|
|
211
|
+
* European currency codes are all consonants: "2297 DKK" read as a Spanish plate
|
|
212
|
+
* 487 times across the corpus.
|
|
213
|
+
*/
|
|
214
|
+
function suppressPlateAsCurrencyAmount(text, match) {
|
|
215
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.LICENSE_PLATE)
|
|
216
|
+
return false;
|
|
217
|
+
if (CURRENCY_CODE_TAIL.test(match.text))
|
|
218
|
+
return true;
|
|
219
|
+
return CURRENCY_CODE_AFTER.test(text.slice(match.end, match.end + 12));
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* A dotted quad is an address, not a tax number. Germany's tax-number shape
|
|
223
|
+
* allows dots between its digit groups, which makes it a superset of IPv4. The
|
|
224
|
+
* address is still redacted — the IP_ADDRESS rule claims the same span — so this
|
|
225
|
+
* only corrects the label.
|
|
226
|
+
*/
|
|
227
|
+
function suppressTaxidAsIpAddress(_text, match) {
|
|
228
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.TAX_ID)
|
|
229
|
+
return false;
|
|
230
|
+
return DOTTED_QUAD.test(match.text.trim());
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Reject two phone shapes that are arithmetic rather than contact details: a
|
|
234
|
+
* year range ("Schooljaar 2025-2026") and the fractional part of a decimal
|
|
235
|
+
* amount ("0.034865 BTC"). Both clear every phone pattern's shape test.
|
|
236
|
+
*/
|
|
237
|
+
function suppressPhoneAsNumberRange(text, match) {
|
|
238
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
|
|
239
|
+
return false;
|
|
240
|
+
if (YEAR_RANGE.test(match.text.trim()))
|
|
241
|
+
return true;
|
|
242
|
+
return DECIMAL_TAIL_BEFORE.test(text.slice(Math.max(0, match.start - 2), match.start));
|
|
243
|
+
}
|
|
37
244
|
function suppressCurrency(text, match) {
|
|
38
245
|
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
39
246
|
return false;
|
|
@@ -52,12 +259,27 @@ function suppressUnits(text, match) {
|
|
|
52
259
|
const [, after] = getContext(text, match.start, match.end);
|
|
53
260
|
return UNIT_AFTER.test(after);
|
|
54
261
|
}
|
|
262
|
+
/**
|
|
263
|
+
* Suppress numbers preceded by reference/invoice/dossier keywords.
|
|
264
|
+
*
|
|
265
|
+
* POSTAL_CODE belongs here for the same reason every other numeric type does,
|
|
266
|
+
* and its absence was an oversight: a five-digit ticket number sits in exactly
|
|
267
|
+
* the shape a German or French postcode occupies, so "Ticket #94730" and
|
|
268
|
+
* "Incident report IR-43433" were masked as addresses.
|
|
269
|
+
*
|
|
270
|
+
* "#" and "XX-" are checked adjacently rather than through the 150-character
|
|
271
|
+
* keyword window. That window is what made the postal rule claim years in
|
|
272
|
+
* dates; widening its reach to fix a different symptom would repeat the error.
|
|
273
|
+
*/
|
|
55
274
|
function suppressReference(text, match) {
|
|
56
|
-
const applicable = new Set([types_js_1.EntityType.PHONE, types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID, types_js_1.EntityType.IBAN, types_js_1.EntityType.CHAMBER_OF_COMMERCE]);
|
|
275
|
+
const applicable = new Set([types_js_1.EntityType.PHONE, types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID, types_js_1.EntityType.IBAN, types_js_1.EntityType.CHAMBER_OF_COMMERCE, types_js_1.EntityType.POSTAL_CODE]);
|
|
57
276
|
if (!applicable.has(match.patternDef.entityType))
|
|
58
277
|
return false;
|
|
59
278
|
const [before] = getContext(text, match.start, match.end);
|
|
60
|
-
|
|
279
|
+
if (REFERENCE_BEFORE.test(before))
|
|
280
|
+
return true;
|
|
281
|
+
const adjacent = text.slice(Math.max(0, match.start - 8), match.start);
|
|
282
|
+
return HASH_BEFORE.test(adjacent) || REF_PREFIX_BEFORE.test(adjacent);
|
|
61
283
|
}
|
|
62
284
|
function suppressLegal(text, match) {
|
|
63
285
|
const applicable = new Set([types_js_1.EntityType.PHONE, types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.POSTAL_CODE]);
|
|
@@ -90,6 +312,16 @@ function suppressYearAsPostal(text, match) {
|
|
|
90
312
|
const clean = match.text.trim();
|
|
91
313
|
if (!RECENT_YEAR.test(clean))
|
|
92
314
|
return false;
|
|
315
|
+
// A date construction *touching* the candidate settles it, whatever the rest
|
|
316
|
+
// of the document says. Without this, the postal-context test below rescued
|
|
317
|
+
// every year in every document with an address in it — and "Adresse", "rue"
|
|
318
|
+
// and "Str." appear in the header of essentially every business letter.
|
|
319
|
+
// Measured: 1,691 of 3,322 postal false positives were plausible years,
|
|
320
|
+
// 1,636 of them literally "2025".
|
|
321
|
+
if (DATE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start)))
|
|
322
|
+
return true;
|
|
323
|
+
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
324
|
+
return true;
|
|
93
325
|
const [before, after] = getContext(text, match.start, match.end);
|
|
94
326
|
const context = before + after;
|
|
95
327
|
// Keep as postal code if postal/address context nearby
|
|
@@ -453,22 +685,59 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
453
685
|
return true;
|
|
454
686
|
return false;
|
|
455
687
|
}
|
|
688
|
+
/**
|
|
689
|
+
* A delimited data row: at least three fields separated by one delimiter.
|
|
690
|
+
* Export formats carry their meaning in the column, not in a nearby word, so a
|
|
691
|
+
* value filling an entire field has structural context even when the line
|
|
692
|
+
* contains no cue.
|
|
693
|
+
*/
|
|
694
|
+
const DELIMITED_ROW = /^[^\n]*?([,;|\t])[^\n]*?\1[^\n]*?\1/;
|
|
695
|
+
/**
|
|
696
|
+
* Types for which filling a delimited field counts as context. Narrow shapes
|
|
697
|
+
* only: applied to every type it was measured a net loss — SECRET gained 1,678
|
|
698
|
+
* false positives, CHAMBER_OF_COMMERCE 456 and POSTAL_CODE 295. A broad shape
|
|
699
|
+
* plus a required cue is a deliberate pairing, and removing the cue leaves only
|
|
700
|
+
* the broad shape.
|
|
701
|
+
*/
|
|
702
|
+
const DELIMITED_FIELD_TYPES = new Set([
|
|
703
|
+
types_js_1.EntityType.PHONE, types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH,
|
|
704
|
+
]);
|
|
705
|
+
/** True when the span is exactly one field of a delimited row. */
|
|
706
|
+
function fillsADelimitedField(text, start, end) {
|
|
707
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
708
|
+
let lineEnd = text.indexOf("\n", end);
|
|
709
|
+
if (lineEnd < 0)
|
|
710
|
+
lineEnd = text.length;
|
|
711
|
+
if (!DELIMITED_ROW.test(text.slice(lineStart, lineEnd)))
|
|
712
|
+
return false;
|
|
713
|
+
const before = text.slice(lineStart, start);
|
|
714
|
+
const after = text.slice(end, lineEnd);
|
|
715
|
+
const opens = before === "" || ",;|\t".includes(before[before.length - 1]);
|
|
716
|
+
const closes = after === "" || ",;|\t".includes(after[0]);
|
|
717
|
+
return opens && closes;
|
|
718
|
+
}
|
|
456
719
|
function suppressRequiresContext(text, match) {
|
|
457
720
|
if (!match.patternDef.requiresContext || match.patternDef.contextKeywords.length === 0)
|
|
458
721
|
return false;
|
|
722
|
+
// A value filling an entire field of a delimited row is introduced by its
|
|
723
|
+
// column, not by a word, so that is context — but only for narrow shapes.
|
|
724
|
+
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
725
|
+
fillsADelimitedField(text, match.start, match.end))
|
|
726
|
+
return false;
|
|
459
727
|
const [before, after] = getContext(text, match.start, match.end);
|
|
460
728
|
const context = (before + " " + after).toLowerCase();
|
|
461
729
|
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
462
730
|
}
|
|
463
731
|
const TYPE_SUPPRESSORS = {
|
|
464
|
-
[types_js_1.EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap],
|
|
732
|
+
[types_js_1.EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
465
733
|
[types_js_1.EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
466
734
|
[types_js_1.EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
467
|
-
[types_js_1.EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
468
|
-
[types_js_1.EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier],
|
|
735
|
+
[types_js_1.EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
736
|
+
[types_js_1.EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier, suppressReference],
|
|
469
737
|
[types_js_1.EntityType.BIC]: [suppressBicWithoutEvidence],
|
|
470
738
|
[types_js_1.EntityType.BANK_ACCOUNT]: [suppressReference],
|
|
471
|
-
[types_js_1.EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict],
|
|
739
|
+
[types_js_1.EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
740
|
+
[types_js_1.EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
472
741
|
[types_js_1.EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
473
742
|
};
|
|
474
743
|
function shouldSuppress(text, match) {
|
|
@@ -191,7 +191,7 @@ const BE = {
|
|
|
191
191
|
p(EntityType.NATIONAL_ID, String.raw `\b\d{2}\.?\d{2}\.?\d{2}[\-.]?\d{3}\.?\d{2}\b`, "belgian_nn"),
|
|
192
192
|
p(EntityType.IBAN, String.raw `\bBE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\b`, "iban"),
|
|
193
193
|
p(EntityType.VAT, String.raw `\bBE\s?0\d{3}\.?\d{3}\.?\d{3}\b`, "belgian_vat"),
|
|
194
|
-
p(EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer"], true),
|
|
194
|
+
p(EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer", "Kruispuntbank", "Banque-Carrefour", "Banque Carrefour", "Ondernemingen", "Entreprises", "onder nummer"], true),
|
|
195
195
|
p(EntityType.PHONE, String.raw `\b0[1-9]\d{0,2}[/\s.\-]?\d{2,3}[.\s\-]?\d{2,3}[.\s\-]?\d{2,3}\b`),
|
|
196
196
|
p(EntityType.PHONE, String.raw `\+32\s?\d{1,3}[\s.\-]?\d{2,3}[\s.\-]?\d{2}[\s.\-]?\d{2}`),
|
|
197
197
|
p(EntityType.PASSPORT, String.raw `\b[A-Z]{2}\d{6}\b`, null, "", ["paspoort", "passport", "passeport", "reisdocument", "travel document", "document de voyage"], true),
|
|
@@ -384,8 +384,15 @@ const ES = {
|
|
|
384
384
|
p(EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}\b`),
|
|
385
385
|
p(EntityType.PHONE, String.raw `\b[679]\d{8}\b`),
|
|
386
386
|
p(EntityType.PHONE, String.raw `\b[679]\d[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
387
|
+
// 3-2-2-2. Just as common as the groupings above and absent until now:
|
|
388
|
+
// "705 97 55 11" matched no phone pattern at all, which cost 674 Spanish
|
|
389
|
+
// numbers — the single largest PHONE gap.
|
|
390
|
+
p(EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
387
391
|
p(EntityType.PHONE, String.raw `\+34\s?[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}`),
|
|
388
|
-
|
|
392
|
+
// A single space, not \s: \s matched a line break, so a year ending one
|
|
393
|
+
// line and a label starting the next ("2002\nCPR") read as one
|
|
394
|
+
// registration — 310 times across the corpus.
|
|
395
|
+
p(EntityType.LICENSE_PLATE, String.raw `\b\d{4}[ ]?[BCDFGHJKLMNPRSTVWXYZ]{3}\b`),
|
|
389
396
|
p(EntityType.POSTAL_CODE, String.raw `\b(?:0[1-9]|[1-4]\d|5[0-2])\d{3}\b`, null, "", ["código postal", "C.P.", "CP", "dirección", "domicilio", "calle", "avenida", "plaza", "paseo", "camino", "Postal:"], true),
|
|
390
397
|
],
|
|
391
398
|
};
|
package/dist/esm/rules/engine.js
CHANGED
|
@@ -53,6 +53,46 @@ function warnUnknownCountry(code) {
|
|
|
53
53
|
`shared country-independent patterns only (email, IBAN, international ` +
|
|
54
54
|
`phone, credit card, ...). Available: ${available.join(", ")}`);
|
|
55
55
|
}
|
|
56
|
+
// ── Local cues: what the document calls the value next to it ────────────
|
|
57
|
+
//
|
|
58
|
+
// Country evidence resolves which *national scheme* owns an ambiguous value, but
|
|
59
|
+
// it never looks at the label sitting immediately in front of it. So
|
|
60
|
+
// "Phone: 0705237535" was reported as a Swedish national ID: the digits pass
|
|
61
|
+
// that checksum, the document is Swedish, and nothing consulted the word
|
|
62
|
+
// "Phone".
|
|
63
|
+
//
|
|
64
|
+
// A cue touching the span is the strongest statement the document makes about
|
|
65
|
+
// what the value *is*, so it outranks the checksum — but only among candidates
|
|
66
|
+
// covering the **same span**, so it decides the label and can never change which
|
|
67
|
+
// characters are masked. That is what keeps invariant I1 intact.
|
|
68
|
+
//
|
|
69
|
+
// Measured: 263 phone numbers and 18 French SIREN were reported as national IDs,
|
|
70
|
+
// each with an explicit cue in front of it.
|
|
71
|
+
const LOCAL_CUES = [
|
|
72
|
+
[EntityType.PHONE,
|
|
73
|
+
/(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell)\w*\s*[:.\-]?\s*$/i],
|
|
74
|
+
[EntityType.CHAMBER_OF_COMMERCE,
|
|
75
|
+
/(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr)\w*\s*[:.\-]?\s*$/i],
|
|
76
|
+
[EntityType.VAT,
|
|
77
|
+
/(?:vat|btw|tva|ust|iva|moms|alv|mwst)[\w.\-]*\s*[:.\-]?\s*$/i],
|
|
78
|
+
[EntityType.BIC,
|
|
79
|
+
/(?:bic|swift)[\w/]*\s*[:.\-]?\s*\(?\s*$/i],
|
|
80
|
+
[EntityType.NATIONAL_ID,
|
|
81
|
+
/(?:bsn|personnummer|rijksregisternummer|national\s*id|identiteitsnummer|henkilötunnus|kennitala|cpr|nif|dni)\w*\s*[:.\-]?\s*$/i],
|
|
82
|
+
];
|
|
83
|
+
// How far back a cue may sit. Short on purpose: a cue is only evidence about the
|
|
84
|
+
// value it introduces, which is the lesson of the postal-code year defect, where
|
|
85
|
+
// a cue 150 characters away licensed every number in the window.
|
|
86
|
+
const CUE_WINDOW = 22;
|
|
87
|
+
/** 1 when a cue naming `entityType` sits immediately before the span. */
|
|
88
|
+
function localCueBonus(text, start, entityType) {
|
|
89
|
+
const before = text.slice(Math.max(0, start - CUE_WINDOW), start);
|
|
90
|
+
for (const [cuedType, pattern] of LOCAL_CUES) {
|
|
91
|
+
if (cuedType === entityType && pattern.test(before))
|
|
92
|
+
return 1;
|
|
93
|
+
}
|
|
94
|
+
return 0;
|
|
95
|
+
}
|
|
56
96
|
export class RuleEngine {
|
|
57
97
|
matcher = new MultiPatternMatcher();
|
|
58
98
|
loadedCountries = new Set();
|
|
@@ -336,6 +376,7 @@ export class RuleEngine {
|
|
|
336
376
|
? match.countryCode : null;
|
|
337
377
|
candidates.push({
|
|
338
378
|
blindPriority,
|
|
379
|
+
cue: localCueBonus(text, match.start, match.patternDef.entityType),
|
|
339
380
|
priority,
|
|
340
381
|
length: match.end - match.start,
|
|
341
382
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
@@ -353,7 +394,7 @@ export class RuleEngine {
|
|
|
353
394
|
});
|
|
354
395
|
}
|
|
355
396
|
for (const d of detectStructuralDob(text)) {
|
|
356
|
-
candidates.push({ blindPriority: 1, priority: 1, length: d.end - d.start,
|
|
397
|
+
candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
|
|
357
398
|
countryScore: 0, inScope: 1, det: d });
|
|
358
399
|
}
|
|
359
400
|
const detections = this.deduplicate(candidates);
|
|
@@ -416,9 +457,10 @@ export class RuleEngine {
|
|
|
416
457
|
const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first } country-blind
|
|
417
458
|
tierOf(b) - tierOf(a) || // best tier here } — decides
|
|
418
459
|
a.det.start - b.det.start || // leftmost } WHICH span
|
|
419
|
-
b.
|
|
420
|
-
b.
|
|
421
|
-
b.
|
|
460
|
+
b.cue - a.cue || // local cue } country-aware
|
|
461
|
+
b.priority - a.priority || // country priority } — decides the
|
|
462
|
+
b.countryScore - a.countryScore || // country evidence } LABEL only
|
|
463
|
+
b.inScope - a.inScope); // declared scope }
|
|
422
464
|
const result = [];
|
|
423
465
|
const occupied = new Set();
|
|
424
466
|
for (const { det } of sorted) {
|
|
@@ -9,14 +9,29 @@ function getContext(text, start, end) {
|
|
|
9
9
|
}
|
|
10
10
|
// Unicode-safe word boundary: negative lookahead for any letter (including accented)
|
|
11
11
|
const _UWB = "(?![a-zA-Z\\u00C0-\\u024F\\u0400-\\u04FF])";
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
// Symbols are held apart from the alphabetic codes because a word boundary
|
|
13
|
+
// after "€" is meaningless — the symbol is not a word character. Python used a
|
|
14
|
+
// bare "\b" here and so could never match "1163 €," at all; this port used the
|
|
15
|
+
// lookahead below and did not have that defect. Kept split so the difference
|
|
16
|
+
// cannot come back.
|
|
17
|
+
const CURRENCY_SYMBOL = "€|\\$|£|¥|₺|zł|Kč|лв|kn|Ft|₽";
|
|
18
|
+
const CURRENCY_WORD = "EUR|USD|GBP|CHF|ISK|SEK|NOK|DKK|PLN|CZK|HUF|RON|BGN|HRK|" +
|
|
19
|
+
"euro|euros|dollar|dollars|pond|kronor|kroner|kronur|kr|" +
|
|
20
|
+
"złoty|korun|forint|lei|leva";
|
|
21
|
+
const CURRENCY_AFTER = new RegExp(`^\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
22
|
+
const CURRENCY_COMMA_AFTER = new RegExp(`^[.,]\\d{1,2}\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
23
|
+
const CURRENCY_BEFORE = new RegExp(`(?:(?:${CURRENCY_SYMBOL})|\\b(?:${CURRENCY_WORD}))\\s*$`, "i");
|
|
24
|
+
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)\s*(?:\w+\s*)?:?\s*$/i;
|
|
16
25
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
17
26
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
18
27
|
const UNIT_AFTER = /^\s*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
19
|
-
|
|
28
|
+
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
29
|
+
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|ticket|incident\s*(?:report|nr|no)?|case\s*(?:nr|no|id)?|zaaknummer|meldingsnummer|Vorgangsnummer|Vorgang|Störungsmeldung|saksnummer|ärendenummer|sagsnummer|asianumero|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.#]?\s*$/i;
|
|
30
|
+
// "#" before a number marks a reference in every language in scope — a ticket,
|
|
31
|
+
// an order, a line item — and never a postcode or a national identifier.
|
|
32
|
+
const HASH_BEFORE = /#\s*$/;
|
|
33
|
+
// A short uppercase tag hyphenated to the number: "IR-43433", "INC-2024".
|
|
34
|
+
const REF_PREFIX_BEFORE = /(?:^|[\s([])[A-Z]{2,5}-$/;
|
|
20
35
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
21
36
|
const MATH_BEFORE = /[=+\-×÷*/]\s*$/;
|
|
22
37
|
const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
@@ -25,12 +40,204 @@ const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
|
25
40
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
26
41
|
const SEQUENTIAL_PATTERNS = /^(?:0{6,}|1234567890?|0123456789|9876543210?|1111111111?|000000000|123456789)$/;
|
|
27
42
|
// YEAR_PATTERN kept for reference but replaced by RECENT_YEAR in suppressYearAsPostal
|
|
28
|
-
|
|
43
|
+
// The leading \b is load-bearing. Without it the short alternatives match the
|
|
44
|
+
// *tail* of an ordinary word: "NIS" matched "tālru**nis** ", the Latvian for
|
|
45
|
+
// "telephone", so every Latvian phone number was suppressed as though it sat
|
|
46
|
+
// behind a Belgian national-ID label. 653 misses from a missing word boundary.
|
|
47
|
+
const ID_LABEL_BEFORE = /\b(?:BSN|RR|NN|NIR|INSZ|NISS|NIS|Steuer-?ID|TIN|NIF|NIE|SSN|rijksregisternummer|numéro\s*national|national\s*number|matricule|Ausweisnummer|Personalausweis|Versichertennummer|KVNR|KV-Nr|Steuernummer|St\.\-Nr|StNr|Finanzamt\s+ist|Ondernemingen\s+onder\s+nummer|ondernemingsnummer|numéro\s*d'entreprise|enterprise\s*number|Kruispuntbank)\s*[:.]?\s*$/i;
|
|
29
48
|
const SERVICE_NUMBER = /^0800[\-\s]/;
|
|
30
49
|
const DATE_PATTERN_FULL = /^\d{2}[-/.]\d{2}[-/.]\d{4}$|^\d{4}[-/.]\d{2}[-/.]\d{2}$/;
|
|
31
50
|
const PASSPORT_CONTEXT_BEFORE = /(?:Reisepass|passport|passeport|paspoort|Bisheriger\s+Reisepass)\s*(?:Nr\.?|Nummer|nummer|number|n[°o])?\s*[:.]?\s*$/i;
|
|
32
51
|
const SE_ORG_CONTEXT_BEFORE = /(?:org\.?\s*nr\.?|organisationsnummer|organisationsnr|Bolagsverket|företag)\s*[:.]?\s*$/i;
|
|
33
52
|
const NUMERIC_TYPES = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID, EntityType.POSTAL_CODE]);
|
|
53
|
+
// ── Date adjacency, for the postal-code year gate ───────────────────────
|
|
54
|
+
//
|
|
55
|
+
// Month names in the languages this engine covers. Long, but it is data: the
|
|
56
|
+
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
57
|
+
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
58
|
+
const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
59
|
+
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
60
|
+
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
61
|
+
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
62
|
+
"|januari|februari|maart|mei|augustus|mrt" +
|
|
63
|
+
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
64
|
+
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
65
|
+
"|janeiro|fevereiro|março|marco|maio|junho|julho|setembro|outubro|novembro|dezembro" +
|
|
66
|
+
// "desember" (nb/nn/is) was the one December spelling missing from a list
|
|
67
|
+
// that already carried ten others — "1. desember 2025" was a Norwegian
|
|
68
|
+
// postcode because of it.
|
|
69
|
+
"|marts|augusti|desember" +
|
|
70
|
+
"|janúar|febrúar|apríl|maí|júní|júlí|ágúst|nóvember" +
|
|
71
|
+
"|tammikuu|helmikuu|maaliskuu|huhtikuu|toukokuu|kesäkuu|heinäkuu|elokuu|syyskuu|lokakuu|marraskuu|joulukuu" +
|
|
72
|
+
"|jaanuar|veebruar|aprill|juuni|juuli|oktoober|detsember" +
|
|
73
|
+
"|janvāris|februāris|aprīlis|maijs|jūnijs|jūlijs|augusts|septembris|oktobris|novembris|decembris" +
|
|
74
|
+
"|stycznia|lutego|marca|kwietnia|maja|czerwca|lipca|sierpnia|września|października|listopada|grudnia" +
|
|
75
|
+
"|ledna|února|března|dubna|května|června|července|srpna|září|října|listopadu|prosince" +
|
|
76
|
+
"|marec|junij|julij|avgust|siječnja|veljače|ožujka|travnja|svibnja|lipnja|srpnja|kolovoza|rujna|studenoga|prosinca" +
|
|
77
|
+
"|március|április|május|június|július|augusztus|szeptember|október" +
|
|
78
|
+
"|ianuarie|februarie|martie|aprilie|iunie|iulie|septembrie|octombrie|noiembrie|decembrie" +
|
|
79
|
+
"|ιανουαρίου|φεβρουαρίου|μαρτίου|απριλίου|μαΐου|ιουνίου|ιουλίου|αυγούστου|σεπτεμβρίου|οκτωβρίου|νοεμβρίου|δεκεμβρίου" +
|
|
80
|
+
"|януари|февруари|март|април|май|юни|юли|август|септември|октомври|ноември|декември";
|
|
81
|
+
/** "since" cues, which introduce a bare year in every one of these languages. */
|
|
82
|
+
const SINCE_CUES = "seit|since|depuis|sinds|sedert|vanaf|desde|dal|od|sedan|siden|alates|alkaen|από|от|din|iz|ab";
|
|
83
|
+
/**
|
|
84
|
+
* A month name, a "since" cue, or a numeric date tail immediately before the
|
|
85
|
+
* candidate. Adjacent-only on purpose — a cue 150 characters away is what made
|
|
86
|
+
* this a bug in the first place.
|
|
87
|
+
*/
|
|
88
|
+
const DATE_BEFORE = new RegExp(`(?:\\b(?:${MONTH_NAMES})\\b\\.?|\\b(?:${SINCE_CUES})\\b|\\d{1,2}[./-]\\d{1,2}[./-])\\s*$`, "iu");
|
|
89
|
+
/** A date separator immediately after, making the candidate a leading year. */
|
|
90
|
+
const DATE_AFTER = /^[./-]\d{1,2}[./-]\d{1,2}\b/;
|
|
91
|
+
/**
|
|
92
|
+
* ISO 4217 codes made only of the consonants a Spanish plate accepts, so a money
|
|
93
|
+
* amount reads as a registration. This is why "2297 DKK" was a plate.
|
|
94
|
+
*/
|
|
95
|
+
// Crypto tickers are the same shape and the same mistake — every surviving
|
|
96
|
+
// plate false positive was "4499 BTC" or a sibling. A ticker is a unit, exactly
|
|
97
|
+
// as an ISO 4217 code is.
|
|
98
|
+
const CURRENCY_CODE = "(?:DKK|SEK|NOK|ISK|CZK|PLN|HUF|RON|BGN|HRK|CHF|GBP|TRY|RSD|MKD|BYN|KZT|CNY|JPY|KRW|ZAR|BRL|MXN|CLP|COP|PLZ|SKK|TRL" +
|
|
99
|
+
"|BTC|ETH|LTC|XRP|BCH|XLM|XMR|DOT|SOL|ADA|TRX|DOGE|USDT|USDC)";
|
|
100
|
+
const CURRENCY_CODE_TAIL = new RegExp(`\\s${CURRENCY_CODE}$`);
|
|
101
|
+
const CURRENCY_CODE_AFTER = new RegExp(`^\\s*${CURRENCY_CODE}\\b`);
|
|
102
|
+
/** Log-timestamp tails, which the ":"-anchored secret rule picks up. */
|
|
103
|
+
const TIMESTAMP_FRAGMENT = /^\d{1,2}:\d{2}(?::\d{2})?(?:[.,]\d{1,6})?Z?$/;
|
|
104
|
+
/** Cloud region names and similar well-known configuration values. */
|
|
105
|
+
const NOT_A_SECRET = new RegExp("^(?:af|ap|ca|cn|eu|il|me|sa|us|gov)-(?:north|south|east|west|central" +
|
|
106
|
+
"|northeast|northwest|southeast|southwest)-\\d[a-z]?$" +
|
|
107
|
+
"|^(?:application|text|image|audio|video|multipart)/[\\w.+-]+$" +
|
|
108
|
+
"|^(?:utf|iso|windows)-[\\d-]+$" +
|
|
109
|
+
"|^(?:no-cache|no-store|max-age|gzip|deflate|identity|chunked)\\b", "i");
|
|
110
|
+
const EXACT_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
|
|
111
|
+
const BIC_SHAPE = /^[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?$/;
|
|
112
|
+
const BIC_CUE_BEFORE = /(?:BIC|SWIFT|BIC\/SWIFT)\s*[:=]?\s*\(?\s*$/i;
|
|
113
|
+
/** An email address, exactly. Same argument as the UUID and the BIC. */
|
|
114
|
+
const EXACT_EMAIL = /^[\w.!#$%&'*+/=?^`{|}~-]+@[\w-]+(?:\.[\w-]+)+$/;
|
|
115
|
+
/**
|
|
116
|
+
* A URL, or the tail of one after the scheme has been split off. The secret
|
|
117
|
+
* rules anchor on ":" and "=", so "https://api.sendgrid.com/v3/mail/send" hands
|
|
118
|
+
* them "//api.sendgrid.com/v3/mail/send" and a slash-separated path scores as
|
|
119
|
+
* high-entropy. An endpoint is published documentation, not a credential.
|
|
120
|
+
*/
|
|
121
|
+
const URL_LIKE = /^(?:[a-z][a-z0-9+.-]*:)?\/\/[^\s/]+(?:\/\S*)?$|^[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z]{2,}(?::\d{1,5})?(?:\/\S*)?$/i;
|
|
122
|
+
/**
|
|
123
|
+
* A URL that *carries* a credential is the opposite case, and it is the common
|
|
124
|
+
* one: "mongodb://admin:DFKDKi1eb51OOhuHPYz@rds-main.eu-west-1.rds.amazonaws.com"
|
|
125
|
+
* is a connection string with a live password in it. Suppressing those as "just
|
|
126
|
+
* a URL" lost 347 real secrets, so the endpoint test must not fire when userinfo
|
|
127
|
+
* or a credential-bearing query parameter is present.
|
|
128
|
+
*/
|
|
129
|
+
const URL_WITH_CREDENTIALS = /^(?:[a-z][a-z0-9+.-]*:)?\/\/[^/@\s]*:[^/@\s]*@|[?&](?:api_?key|access_?token|auth|token|secret|password|pwd|sig|signature|credential)=/i;
|
|
130
|
+
/** An LDAP distinguished name: "cn=github-actions,dc=corp,dc=eu". */
|
|
131
|
+
const LDAP_DN = /^(?:cn|ou|dc|uid|o|l|st|c)=[^,=]+(?:,\s*(?:cn|ou|dc|uid|o|l|st|c)=[^,=]+)+$/i;
|
|
132
|
+
/**
|
|
133
|
+
* Lower-case words joined by hyphens, optionally with a short id suffix:
|
|
134
|
+
* "service-account", "data-lake-33e061". The single-word test below already
|
|
135
|
+
* rejects "sozialversicherungsnummer" on the reasoning that a real secret would
|
|
136
|
+
* be mixed case; a hyphenated compound is the same token with a separator.
|
|
137
|
+
*/
|
|
138
|
+
const HYPHENATED_WORDS = /^[a-z]{2,}(?:-[a-z]{2,})+(?:-[0-9a-f]{4,8})?$/;
|
|
139
|
+
/** A dotted quad with every octet in range. */
|
|
140
|
+
const DOTTED_QUAD = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
|
|
141
|
+
/** Two plausible years joined by a hyphen — a school year, a contract term. */
|
|
142
|
+
const YEAR_RANGE = /^(19|20)\d{2}\s?[-–/]\s?(19|20)\d{2}$/;
|
|
143
|
+
/** A digit and a decimal point immediately before, so the match is a fraction. */
|
|
144
|
+
const DECIMAL_TAIL_BEFORE = /\d[.,]$/;
|
|
145
|
+
/**
|
|
146
|
+
* Punctuation the greedy "[^\s]{8,}" secret rule sweeps up with the token: it
|
|
147
|
+
* matched "TRIONL2U)." out of "(BIC: TRIONL2U).". Stripped only for the shape
|
|
148
|
+
* tests — the emitted span is never altered here.
|
|
149
|
+
*/
|
|
150
|
+
const PUNCT_EDGES = /^[\s([{<"'`]+|[\s)\]}>"'`.,;:!?]+$/g;
|
|
151
|
+
function coreToken(raw) {
|
|
152
|
+
return raw.replace(PUNCT_EDGES, "").replace(PUNCT_EDGES, "");
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* A generic secret must not claim a span that is a specific known type.
|
|
156
|
+
*
|
|
157
|
+
* The high-entropy rules are deliberately broad and they carry a validator, so
|
|
158
|
+
* they reach the top priority tier while the structured detector for the same
|
|
159
|
+
* characters sits at the bottom with nothing to offer. Measured on 152,300
|
|
160
|
+
* documents: 687 UUIDs and 140 BICs were reported as SECRET, each counted twice
|
|
161
|
+
* over — a false positive for SECRET and a miss for the type that should have
|
|
162
|
+
* had it.
|
|
163
|
+
*/
|
|
164
|
+
function suppressSecretOverStructured(text, match) {
|
|
165
|
+
if (match.patternDef.entityType !== EntityType.SECRET)
|
|
166
|
+
return false;
|
|
167
|
+
const token = coreToken(match.text);
|
|
168
|
+
if (EXACT_UUID.test(token) || EXACT_EMAIL.test(token))
|
|
169
|
+
return true;
|
|
170
|
+
if (BIC_SHAPE.test(token)) {
|
|
171
|
+
return BIC_CUE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
|
|
172
|
+
}
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Reject the three things the ":"-anchored secret rule reliably mistakes:
|
|
177
|
+
* timestamps ("57:22.283Z]"), ordinary words ("Sozialversicherungsnummer") and
|
|
178
|
+
* region names ("us-east-1"). All three sit after a colon or equals sign, which
|
|
179
|
+
* is all that rule asks for, and all three clear the entropy threshold.
|
|
180
|
+
*
|
|
181
|
+
* The word test is narrow on purpose: purely alphabetic *and* shaped like one
|
|
182
|
+
* natural word. A random all-letter token would be mixed case and is left alone
|
|
183
|
+
* — losing a real secret is the expensive direction.
|
|
184
|
+
*/
|
|
185
|
+
function suppressSecretNotASecret(_text, match) {
|
|
186
|
+
if (match.patternDef.entityType !== EntityType.SECRET)
|
|
187
|
+
return false;
|
|
188
|
+
const bare = coreToken(match.text);
|
|
189
|
+
if (TIMESTAMP_FRAGMENT.test(bare))
|
|
190
|
+
return true;
|
|
191
|
+
if (NOT_A_SECRET.test(bare))
|
|
192
|
+
return true;
|
|
193
|
+
if (URL_LIKE.test(bare) && !URL_WITH_CREDENTIALS.test(bare))
|
|
194
|
+
return true;
|
|
195
|
+
if (LDAP_DN.test(bare))
|
|
196
|
+
return true;
|
|
197
|
+
if (HYPHENATED_WORDS.test(bare))
|
|
198
|
+
return true;
|
|
199
|
+
const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
|
|
200
|
+
const oneWord = bare === bare.toLowerCase() ||
|
|
201
|
+
(bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
|
|
202
|
+
bare.slice(1) === bare.slice(1).toLowerCase());
|
|
203
|
+
return alpha && oneWord;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* A money amount followed by its ISO 4217 code is not a registration. Spain's
|
|
207
|
+
* plate shape is four digits then three consonants, and the Nordic and Central
|
|
208
|
+
* European currency codes are all consonants: "2297 DKK" read as a Spanish plate
|
|
209
|
+
* 487 times across the corpus.
|
|
210
|
+
*/
|
|
211
|
+
function suppressPlateAsCurrencyAmount(text, match) {
|
|
212
|
+
if (match.patternDef.entityType !== EntityType.LICENSE_PLATE)
|
|
213
|
+
return false;
|
|
214
|
+
if (CURRENCY_CODE_TAIL.test(match.text))
|
|
215
|
+
return true;
|
|
216
|
+
return CURRENCY_CODE_AFTER.test(text.slice(match.end, match.end + 12));
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* A dotted quad is an address, not a tax number. Germany's tax-number shape
|
|
220
|
+
* allows dots between its digit groups, which makes it a superset of IPv4. The
|
|
221
|
+
* address is still redacted — the IP_ADDRESS rule claims the same span — so this
|
|
222
|
+
* only corrects the label.
|
|
223
|
+
*/
|
|
224
|
+
function suppressTaxidAsIpAddress(_text, match) {
|
|
225
|
+
if (match.patternDef.entityType !== EntityType.TAX_ID)
|
|
226
|
+
return false;
|
|
227
|
+
return DOTTED_QUAD.test(match.text.trim());
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Reject two phone shapes that are arithmetic rather than contact details: a
|
|
231
|
+
* year range ("Schooljaar 2025-2026") and the fractional part of a decimal
|
|
232
|
+
* amount ("0.034865 BTC"). Both clear every phone pattern's shape test.
|
|
233
|
+
*/
|
|
234
|
+
function suppressPhoneAsNumberRange(text, match) {
|
|
235
|
+
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
236
|
+
return false;
|
|
237
|
+
if (YEAR_RANGE.test(match.text.trim()))
|
|
238
|
+
return true;
|
|
239
|
+
return DECIMAL_TAIL_BEFORE.test(text.slice(Math.max(0, match.start - 2), match.start));
|
|
240
|
+
}
|
|
34
241
|
function suppressCurrency(text, match) {
|
|
35
242
|
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
36
243
|
return false;
|
|
@@ -49,12 +256,27 @@ function suppressUnits(text, match) {
|
|
|
49
256
|
const [, after] = getContext(text, match.start, match.end);
|
|
50
257
|
return UNIT_AFTER.test(after);
|
|
51
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* Suppress numbers preceded by reference/invoice/dossier keywords.
|
|
261
|
+
*
|
|
262
|
+
* POSTAL_CODE belongs here for the same reason every other numeric type does,
|
|
263
|
+
* and its absence was an oversight: a five-digit ticket number sits in exactly
|
|
264
|
+
* the shape a German or French postcode occupies, so "Ticket #94730" and
|
|
265
|
+
* "Incident report IR-43433" were masked as addresses.
|
|
266
|
+
*
|
|
267
|
+
* "#" and "XX-" are checked adjacently rather than through the 150-character
|
|
268
|
+
* keyword window. That window is what made the postal rule claim years in
|
|
269
|
+
* dates; widening its reach to fix a different symptom would repeat the error.
|
|
270
|
+
*/
|
|
52
271
|
function suppressReference(text, match) {
|
|
53
|
-
const applicable = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID, EntityType.IBAN, EntityType.CHAMBER_OF_COMMERCE]);
|
|
272
|
+
const applicable = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID, EntityType.IBAN, EntityType.CHAMBER_OF_COMMERCE, EntityType.POSTAL_CODE]);
|
|
54
273
|
if (!applicable.has(match.patternDef.entityType))
|
|
55
274
|
return false;
|
|
56
275
|
const [before] = getContext(text, match.start, match.end);
|
|
57
|
-
|
|
276
|
+
if (REFERENCE_BEFORE.test(before))
|
|
277
|
+
return true;
|
|
278
|
+
const adjacent = text.slice(Math.max(0, match.start - 8), match.start);
|
|
279
|
+
return HASH_BEFORE.test(adjacent) || REF_PREFIX_BEFORE.test(adjacent);
|
|
58
280
|
}
|
|
59
281
|
function suppressLegal(text, match) {
|
|
60
282
|
const applicable = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.POSTAL_CODE]);
|
|
@@ -87,6 +309,16 @@ function suppressYearAsPostal(text, match) {
|
|
|
87
309
|
const clean = match.text.trim();
|
|
88
310
|
if (!RECENT_YEAR.test(clean))
|
|
89
311
|
return false;
|
|
312
|
+
// A date construction *touching* the candidate settles it, whatever the rest
|
|
313
|
+
// of the document says. Without this, the postal-context test below rescued
|
|
314
|
+
// every year in every document with an address in it — and "Adresse", "rue"
|
|
315
|
+
// and "Str." appear in the header of essentially every business letter.
|
|
316
|
+
// Measured: 1,691 of 3,322 postal false positives were plausible years,
|
|
317
|
+
// 1,636 of them literally "2025".
|
|
318
|
+
if (DATE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start)))
|
|
319
|
+
return true;
|
|
320
|
+
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
321
|
+
return true;
|
|
90
322
|
const [before, after] = getContext(text, match.start, match.end);
|
|
91
323
|
const context = before + after;
|
|
92
324
|
// Keep as postal code if postal/address context nearby
|
|
@@ -450,22 +682,59 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
450
682
|
return true;
|
|
451
683
|
return false;
|
|
452
684
|
}
|
|
685
|
+
/**
|
|
686
|
+
* A delimited data row: at least three fields separated by one delimiter.
|
|
687
|
+
* Export formats carry their meaning in the column, not in a nearby word, so a
|
|
688
|
+
* value filling an entire field has structural context even when the line
|
|
689
|
+
* contains no cue.
|
|
690
|
+
*/
|
|
691
|
+
const DELIMITED_ROW = /^[^\n]*?([,;|\t])[^\n]*?\1[^\n]*?\1/;
|
|
692
|
+
/**
|
|
693
|
+
* Types for which filling a delimited field counts as context. Narrow shapes
|
|
694
|
+
* only: applied to every type it was measured a net loss — SECRET gained 1,678
|
|
695
|
+
* false positives, CHAMBER_OF_COMMERCE 456 and POSTAL_CODE 295. A broad shape
|
|
696
|
+
* plus a required cue is a deliberate pairing, and removing the cue leaves only
|
|
697
|
+
* the broad shape.
|
|
698
|
+
*/
|
|
699
|
+
const DELIMITED_FIELD_TYPES = new Set([
|
|
700
|
+
EntityType.PHONE, EntityType.DOB, EntityType.DATE_OF_DEATH,
|
|
701
|
+
]);
|
|
702
|
+
/** True when the span is exactly one field of a delimited row. */
|
|
703
|
+
function fillsADelimitedField(text, start, end) {
|
|
704
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
705
|
+
let lineEnd = text.indexOf("\n", end);
|
|
706
|
+
if (lineEnd < 0)
|
|
707
|
+
lineEnd = text.length;
|
|
708
|
+
if (!DELIMITED_ROW.test(text.slice(lineStart, lineEnd)))
|
|
709
|
+
return false;
|
|
710
|
+
const before = text.slice(lineStart, start);
|
|
711
|
+
const after = text.slice(end, lineEnd);
|
|
712
|
+
const opens = before === "" || ",;|\t".includes(before[before.length - 1]);
|
|
713
|
+
const closes = after === "" || ",;|\t".includes(after[0]);
|
|
714
|
+
return opens && closes;
|
|
715
|
+
}
|
|
453
716
|
function suppressRequiresContext(text, match) {
|
|
454
717
|
if (!match.patternDef.requiresContext || match.patternDef.contextKeywords.length === 0)
|
|
455
718
|
return false;
|
|
719
|
+
// A value filling an entire field of a delimited row is introduced by its
|
|
720
|
+
// column, not by a word, so that is context — but only for narrow shapes.
|
|
721
|
+
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
722
|
+
fillsADelimitedField(text, match.start, match.end))
|
|
723
|
+
return false;
|
|
456
724
|
const [before, after] = getContext(text, match.start, match.end);
|
|
457
725
|
const context = (before + " " + after).toLowerCase();
|
|
458
726
|
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
459
727
|
}
|
|
460
728
|
const TYPE_SUPPRESSORS = {
|
|
461
|
-
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap],
|
|
729
|
+
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
462
730
|
[EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
463
731
|
[EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
464
|
-
[EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
465
|
-
[EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier],
|
|
732
|
+
[EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
733
|
+
[EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier, suppressReference],
|
|
466
734
|
[EntityType.BIC]: [suppressBicWithoutEvidence],
|
|
467
735
|
[EntityType.BANK_ACCOUNT]: [suppressReference],
|
|
468
|
-
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict],
|
|
736
|
+
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
737
|
+
[EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
469
738
|
[EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
470
739
|
};
|
|
471
740
|
export function shouldSuppress(text, match) {
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.6",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
|
+
"homepage": "https://euredact.dev",
|
|
5
6
|
"license": "Apache-2.0",
|
|
6
7
|
"type": "module",
|
|
7
8
|
"main": "./dist/cjs/index.js",
|
|
@@ -66,5 +67,8 @@
|
|
|
66
67
|
"devDependencies": {
|
|
67
68
|
"typescript": "^5.4.0",
|
|
68
69
|
"tsx": "^4.7.0"
|
|
70
|
+
},
|
|
71
|
+
"bugs": {
|
|
72
|
+
"url": "https://github.com/euredact/euredact/issues"
|
|
69
73
|
}
|
|
70
74
|
}
|