euredact 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -194,7 +194,7 @@ const BE = {
|
|
|
194
194
|
p(types_js_1.EntityType.NATIONAL_ID, String.raw `\b\d{2}\.?\d{2}\.?\d{2}[\-.]?\d{3}\.?\d{2}\b`, "belgian_nn"),
|
|
195
195
|
p(types_js_1.EntityType.IBAN, String.raw `\bBE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\b`, "iban"),
|
|
196
196
|
p(types_js_1.EntityType.VAT, String.raw `\bBE\s?0\d{3}\.?\d{3}\.?\d{3}\b`, "belgian_vat"),
|
|
197
|
-
p(types_js_1.EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer"], true),
|
|
197
|
+
p(types_js_1.EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer", "Kruispuntbank", "Banque-Carrefour", "Banque Carrefour", "Ondernemingen", "Entreprises", "onder nummer"], true),
|
|
198
198
|
p(types_js_1.EntityType.PHONE, String.raw `\b0[1-9]\d{0,2}[/\s.\-]?\d{2,3}[.\s\-]?\d{2,3}[.\s\-]?\d{2,3}\b`),
|
|
199
199
|
p(types_js_1.EntityType.PHONE, String.raw `\+32\s?\d{1,3}[\s.\-]?\d{2,3}[\s.\-]?\d{2}[\s.\-]?\d{2}`),
|
|
200
200
|
p(types_js_1.EntityType.PASSPORT, String.raw `\b[A-Z]{2}\d{6}\b`, null, "", ["paspoort", "passport", "passeport", "reisdocument", "travel document", "document de voyage"], true),
|
|
@@ -226,7 +226,7 @@ const DE = {
|
|
|
226
226
|
p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b[A-ZÄÖÜ]{1,3}[\s\-][A-Z]{1,2}[\s\-]?\d{1,4}[EH]?\b`),
|
|
227
227
|
p(types_js_1.EntityType.POSTAL_CODE, String.raw `\b\d{5}\b`, null, "", ["PLZ", "Postleitzahl", "postal code", "postcode", "Anschrift", "Adresse", "Wohnort", "Ort", "Stadt", "wohnt", "wohnhaft", "Straße", "Str.", "Weg", "Platz", "Allee", "Ring"], true),
|
|
228
228
|
p(types_js_1.EntityType.CHAMBER_OF_COMMERCE, String.raw `\bHR[AB]\s?\d{4,6}\b`),
|
|
229
|
-
p(types_js_1.EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer"], true),
|
|
229
|
+
p(types_js_1.EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer", "SVNR", "SV-Nr", "RVNR"], true),
|
|
230
230
|
p(types_js_1.EntityType.TAX_ID, String.raw `\b\d{2,3}/\d{3,4}/\d{4,5}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
231
231
|
p(types_js_1.EntityType.TAX_ID, String.raw `\b\d{13}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
232
232
|
p(types_js_1.EntityType.HEALTH_INSURANCE, String.raw `\b[A-Z]\d{9}\b`, null, "", ["Versichertennummer", "Krankenversicherung", "KV-Nummer", "Krankenkasse", "GKV", "Versichertenkarte", "Versicherter", "Versicherte", "KVNR", "Krankenversichertennummer"], true),
|
|
@@ -387,8 +387,15 @@ const ES = {
|
|
|
387
387
|
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}\b`),
|
|
388
388
|
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d{8}\b`),
|
|
389
389
|
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
390
|
+
// 3-2-2-2. Just as common as the groupings above and absent until now:
|
|
391
|
+
// "705 97 55 11" matched no phone pattern at all, which cost 674 Spanish
|
|
392
|
+
// numbers — the single largest PHONE gap.
|
|
393
|
+
p(types_js_1.EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
390
394
|
p(types_js_1.EntityType.PHONE, String.raw `\+34\s?[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}`),
|
|
391
|
-
|
|
395
|
+
// A single space, not \s: \s matched a line break, so a year ending one
|
|
396
|
+
// line and a label starting the next ("2002\nCPR") read as one
|
|
397
|
+
// registration — 310 times across the corpus.
|
|
398
|
+
p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b\d{4}[ ]?[BCDFGHJKLMNPRSTVWXYZ]{3}\b`),
|
|
392
399
|
p(types_js_1.EntityType.POSTAL_CODE, String.raw `\b(?:0[1-9]|[1-4]\d|5[0-2])\d{3}\b`, null, "", ["código postal", "C.P.", "CP", "dirección", "domicilio", "calle", "avenida", "plaza", "paseo", "camino", "Postal:"], true),
|
|
393
400
|
],
|
|
394
401
|
};
|
package/dist/cjs/rules/engine.js
CHANGED
|
@@ -90,6 +90,46 @@ function warnUnknownCountry(code) {
|
|
|
90
90
|
`shared country-independent patterns only (email, IBAN, international ` +
|
|
91
91
|
`phone, credit card, ...). Available: ${available.join(", ")}`);
|
|
92
92
|
}
|
|
93
|
+
// ── Local cues: what the document calls the value next to it ────────────
|
|
94
|
+
//
|
|
95
|
+
// Country evidence resolves which *national scheme* owns an ambiguous value, but
|
|
96
|
+
// it never looks at the label sitting immediately in front of it. So
|
|
97
|
+
// "Phone: 0705237535" was reported as a Swedish national ID: the digits pass
|
|
98
|
+
// that checksum, the document is Swedish, and nothing consulted the word
|
|
99
|
+
// "Phone".
|
|
100
|
+
//
|
|
101
|
+
// A cue touching the span is the strongest statement the document makes about
|
|
102
|
+
// what the value *is*, so it outranks the checksum — but only among candidates
|
|
103
|
+
// covering the **same span**, so it decides the label and can never change which
|
|
104
|
+
// characters are masked. That is what keeps invariant I1 intact.
|
|
105
|
+
//
|
|
106
|
+
// Measured: 263 phone numbers and 18 French SIREN were reported as national IDs,
|
|
107
|
+
// each with an explicit cue in front of it.
|
|
108
|
+
const LOCAL_CUES = [
|
|
109
|
+
[types_js_1.EntityType.PHONE,
|
|
110
|
+
/(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell)\w*\s*[:.\-]?\s*$/i],
|
|
111
|
+
[types_js_1.EntityType.CHAMBER_OF_COMMERCE,
|
|
112
|
+
/(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr)\w*\s*[:.\-]?\s*$/i],
|
|
113
|
+
[types_js_1.EntityType.VAT,
|
|
114
|
+
/(?:vat|btw|tva|ust|iva|moms|alv|mwst)[\w.\-]*\s*[:.\-]?\s*$/i],
|
|
115
|
+
[types_js_1.EntityType.BIC,
|
|
116
|
+
/(?:bic|swift)[\w/]*\s*[:.\-]?\s*\(?\s*$/i],
|
|
117
|
+
[types_js_1.EntityType.NATIONAL_ID,
|
|
118
|
+
/(?:bsn|personnummer|rijksregisternummer|national\s*id|identiteitsnummer|henkilötunnus|kennitala|cpr|nif|dni)\w*\s*[:.\-]?\s*$/i],
|
|
119
|
+
];
|
|
120
|
+
// How far back a cue may sit. Short on purpose: a cue is only evidence about the
|
|
121
|
+
// value it introduces, which is the lesson of the postal-code year defect, where
|
|
122
|
+
// a cue 150 characters away licensed every number in the window.
|
|
123
|
+
const CUE_WINDOW = 22;
|
|
124
|
+
/** 1 when a cue naming `entityType` sits immediately before the span. */
|
|
125
|
+
function localCueBonus(text, start, entityType) {
|
|
126
|
+
const before = text.slice(Math.max(0, start - CUE_WINDOW), start);
|
|
127
|
+
for (const [cuedType, pattern] of LOCAL_CUES) {
|
|
128
|
+
if (cuedType === entityType && pattern.test(before))
|
|
129
|
+
return 1;
|
|
130
|
+
}
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
93
133
|
class RuleEngine {
|
|
94
134
|
matcher = new matchers_js_1.MultiPatternMatcher();
|
|
95
135
|
loadedCountries = new Set();
|
|
@@ -373,6 +413,7 @@ class RuleEngine {
|
|
|
373
413
|
? match.countryCode : null;
|
|
374
414
|
candidates.push({
|
|
375
415
|
blindPriority,
|
|
416
|
+
cue: localCueBonus(text, match.start, match.patternDef.entityType),
|
|
376
417
|
priority,
|
|
377
418
|
length: match.end - match.start,
|
|
378
419
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
@@ -390,7 +431,7 @@ class RuleEngine {
|
|
|
390
431
|
});
|
|
391
432
|
}
|
|
392
433
|
for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
|
|
393
|
-
candidates.push({ blindPriority: 1, priority: 1, length: d.end - d.start,
|
|
434
|
+
candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
|
|
394
435
|
countryScore: 0, inScope: 1, det: d });
|
|
395
436
|
}
|
|
396
437
|
const detections = this.deduplicate(candidates);
|
|
@@ -453,9 +494,10 @@ class RuleEngine {
|
|
|
453
494
|
const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first } country-blind
|
|
454
495
|
tierOf(b) - tierOf(a) || // best tier here } — decides
|
|
455
496
|
a.det.start - b.det.start || // leftmost } WHICH span
|
|
456
|
-
b.
|
|
457
|
-
b.
|
|
458
|
-
b.
|
|
497
|
+
b.cue - a.cue || // local cue } country-aware
|
|
498
|
+
b.priority - a.priority || // country priority } — decides the
|
|
499
|
+
b.countryScore - a.countryScore || // country evidence } LABEL only
|
|
500
|
+
b.inScope - a.inScope); // declared scope }
|
|
459
501
|
const result = [];
|
|
460
502
|
const occupied = new Set();
|
|
461
503
|
for (const { det } of sorted) {
|
|
@@ -19,7 +19,8 @@ const AMOUNT_LABEL_BEFORE = /(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|T
|
|
|
19
19
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
20
20
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
21
21
|
const UNIT_AFTER = /^\s*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
22
|
-
|
|
22
|
+
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
23
|
+
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.]?\s*$/i;
|
|
23
24
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
24
25
|
const MATH_BEFORE = /[=+\-×÷*/]\s*$/;
|
|
25
26
|
const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
@@ -28,12 +29,164 @@ const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
|
28
29
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
29
30
|
const SEQUENTIAL_PATTERNS = /^(?:0{6,}|1234567890?|0123456789|9876543210?|1111111111?|000000000|123456789)$/;
|
|
30
31
|
// YEAR_PATTERN kept for reference but replaced by RECENT_YEAR in suppressYearAsPostal
|
|
31
|
-
|
|
32
|
+
// The leading \b is load-bearing. Without it the short alternatives match the
|
|
33
|
+
// *tail* of an ordinary word: "NIS" matched "tālru**nis** ", the Latvian for
|
|
34
|
+
// "telephone", so every Latvian phone number was suppressed as though it sat
|
|
35
|
+
// behind a Belgian national-ID label. 653 misses from a missing word boundary.
|
|
36
|
+
const ID_LABEL_BEFORE = /\b(?:BSN|RR|NN|NIR|INSZ|NISS|NIS|Steuer-?ID|TIN|NIF|NIE|SSN|rijksregisternummer|numéro\s*national|national\s*number|matricule|Ausweisnummer|Personalausweis|Versichertennummer|KVNR|KV-Nr|Steuernummer|St\.\-Nr|StNr|Finanzamt\s+ist|Ondernemingen\s+onder\s+nummer|ondernemingsnummer|numéro\s*d'entreprise|enterprise\s*number|Kruispuntbank)\s*[:.]?\s*$/i;
|
|
32
37
|
const SERVICE_NUMBER = /^0800[\-\s]/;
|
|
33
38
|
const DATE_PATTERN_FULL = /^\d{2}[-/.]\d{2}[-/.]\d{4}$|^\d{4}[-/.]\d{2}[-/.]\d{2}$/;
|
|
34
39
|
const PASSPORT_CONTEXT_BEFORE = /(?:Reisepass|passport|passeport|paspoort|Bisheriger\s+Reisepass)\s*(?:Nr\.?|Nummer|nummer|number|n[°o])?\s*[:.]?\s*$/i;
|
|
35
40
|
const SE_ORG_CONTEXT_BEFORE = /(?:org\.?\s*nr\.?|organisationsnummer|organisationsnr|Bolagsverket|företag)\s*[:.]?\s*$/i;
|
|
36
41
|
const NUMERIC_TYPES = new Set([types_js_1.EntityType.PHONE, types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID, types_js_1.EntityType.POSTAL_CODE]);
|
|
42
|
+
// ── Date adjacency, for the postal-code year gate ───────────────────────
|
|
43
|
+
//
|
|
44
|
+
// Month names in the languages this engine covers. Long, but it is data: the
|
|
45
|
+
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
46
|
+
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
47
|
+
const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
48
|
+
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
49
|
+
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
50
|
+
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
51
|
+
"|januari|februari|maart|mei|augustus|mrt" +
|
|
52
|
+
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
53
|
+
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
54
|
+
"|janeiro|fevereiro|março|marco|maio|junho|julho|setembro|outubro|novembro|dezembro" +
|
|
55
|
+
"|marts|augusti" +
|
|
56
|
+
"|tammikuu|helmikuu|maaliskuu|huhtikuu|toukokuu|kesäkuu|heinäkuu|elokuu|syyskuu|lokakuu|marraskuu|joulukuu" +
|
|
57
|
+
"|jaanuar|veebruar|aprill|juuni|juuli|oktoober|detsember" +
|
|
58
|
+
"|janvāris|februāris|aprīlis|maijs|jūnijs|jūlijs|augusts|septembris|oktobris|novembris|decembris" +
|
|
59
|
+
"|stycznia|lutego|marca|kwietnia|maja|czerwca|lipca|sierpnia|września|października|listopada|grudnia" +
|
|
60
|
+
"|ledna|února|března|dubna|května|června|července|srpna|září|října|listopadu|prosince" +
|
|
61
|
+
"|marec|junij|julij|avgust|siječnja|veljače|ožujka|travnja|svibnja|lipnja|srpnja|kolovoza|rujna|studenoga|prosinca" +
|
|
62
|
+
"|március|április|május|június|július|augusztus|szeptember|október" +
|
|
63
|
+
"|ianuarie|februarie|martie|aprilie|iunie|iulie|septembrie|octombrie|noiembrie|decembrie" +
|
|
64
|
+
"|ιανουαρίου|φεβρουαρίου|μαρτίου|απριλίου|μαΐου|ιουνίου|ιουλίου|αυγούστου|σεπτεμβρίου|οκτωβρίου|νοεμβρίου|δεκεμβρίου" +
|
|
65
|
+
"|януари|февруари|март|април|май|юни|юли|август|септември|октомври|ноември|декември";
|
|
66
|
+
/** "since" cues, which introduce a bare year in every one of these languages. */
|
|
67
|
+
const SINCE_CUES = "seit|since|depuis|sinds|sedert|vanaf|desde|dal|od|sedan|siden|alates|alkaen|από|от|din|iz|ab";
|
|
68
|
+
/**
|
|
69
|
+
* A month name, a "since" cue, or a numeric date tail immediately before the
|
|
70
|
+
* candidate. Adjacent-only on purpose — a cue 150 characters away is what made
|
|
71
|
+
* this a bug in the first place.
|
|
72
|
+
*/
|
|
73
|
+
const DATE_BEFORE = new RegExp(`(?:\\b(?:${MONTH_NAMES})\\b\\.?|\\b(?:${SINCE_CUES})\\b|\\d{1,2}[./-]\\d{1,2}[./-])\\s*$`, "iu");
|
|
74
|
+
/** A date separator immediately after, making the candidate a leading year. */
|
|
75
|
+
const DATE_AFTER = /^[./-]\d{1,2}[./-]\d{1,2}\b/;
|
|
76
|
+
/**
|
|
77
|
+
* ISO 4217 codes made only of the consonants a Spanish plate accepts, so a money
|
|
78
|
+
* amount reads as a registration. This is why "2297 DKK" was a plate.
|
|
79
|
+
*/
|
|
80
|
+
const CURRENCY_CODE = "(?:DKK|SEK|NOK|ISK|CZK|PLN|HUF|RON|BGN|HRK|CHF|GBP|TRY|RSD|MKD|BYN|KZT|CNY|JPY|KRW|ZAR|BRL|MXN|CLP|COP|PLZ|SKK|TRL)";
|
|
81
|
+
const CURRENCY_CODE_TAIL = new RegExp(`\\s${CURRENCY_CODE}$`);
|
|
82
|
+
const CURRENCY_CODE_AFTER = new RegExp(`^\\s*${CURRENCY_CODE}\\b`);
|
|
83
|
+
/** Log-timestamp tails, which the ":"-anchored secret rule picks up. */
|
|
84
|
+
const TIMESTAMP_FRAGMENT = /^\d{1,2}:\d{2}(?::\d{2})?(?:[.,]\d{1,6})?Z?$/;
|
|
85
|
+
/** Cloud region names and similar well-known configuration values. */
|
|
86
|
+
const NOT_A_SECRET = new RegExp("^(?:af|ap|ca|cn|eu|il|me|sa|us|gov)-(?:north|south|east|west|central" +
|
|
87
|
+
"|northeast|northwest|southeast|southwest)-\\d[a-z]?$" +
|
|
88
|
+
"|^(?:application|text|image|audio|video|multipart)/[\\w.+-]+$" +
|
|
89
|
+
"|^(?:utf|iso|windows)-[\\d-]+$" +
|
|
90
|
+
"|^(?:no-cache|no-store|max-age|gzip|deflate|identity|chunked)\\b", "i");
|
|
91
|
+
const EXACT_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
|
|
92
|
+
const BIC_SHAPE = /^[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?$/;
|
|
93
|
+
const BIC_CUE_BEFORE = /(?:BIC|SWIFT|BIC\/SWIFT)\s*[:=]?\s*\(?\s*$/i;
|
|
94
|
+
/** A dotted quad with every octet in range. */
|
|
95
|
+
const DOTTED_QUAD = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
|
|
96
|
+
/** Two plausible years joined by a hyphen — a school year, a contract term. */
|
|
97
|
+
const YEAR_RANGE = /^(19|20)\d{2}\s?[-–/]\s?(19|20)\d{2}$/;
|
|
98
|
+
/** A digit and a decimal point immediately before, so the match is a fraction. */
|
|
99
|
+
const DECIMAL_TAIL_BEFORE = /\d[.,]$/;
|
|
100
|
+
/**
|
|
101
|
+
* Punctuation the greedy "[^\s]{8,}" secret rule sweeps up with the token: it
|
|
102
|
+
* matched "TRIONL2U)." out of "(BIC: TRIONL2U).". Stripped only for the shape
|
|
103
|
+
* tests — the emitted span is never altered here.
|
|
104
|
+
*/
|
|
105
|
+
const PUNCT_EDGES = /^[\s([{<"'`]+|[\s)\]}>"'`.,;:!?]+$/g;
|
|
106
|
+
function coreToken(raw) {
|
|
107
|
+
return raw.replace(PUNCT_EDGES, "").replace(PUNCT_EDGES, "");
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* A generic secret must not claim a span that is a specific known type.
|
|
111
|
+
*
|
|
112
|
+
* The high-entropy rules are deliberately broad and they carry a validator, so
|
|
113
|
+
* they reach the top priority tier while the structured detector for the same
|
|
114
|
+
* characters sits at the bottom with nothing to offer. Measured on 152,300
|
|
115
|
+
* documents: 687 UUIDs and 140 BICs were reported as SECRET, each counted twice
|
|
116
|
+
* over — a false positive for SECRET and a miss for the type that should have
|
|
117
|
+
* had it.
|
|
118
|
+
*/
|
|
119
|
+
function suppressSecretOverStructured(text, match) {
|
|
120
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.SECRET)
|
|
121
|
+
return false;
|
|
122
|
+
const token = coreToken(match.text);
|
|
123
|
+
if (EXACT_UUID.test(token))
|
|
124
|
+
return true;
|
|
125
|
+
if (BIC_SHAPE.test(token)) {
|
|
126
|
+
return BIC_CUE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
|
|
127
|
+
}
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Reject the three things the ":"-anchored secret rule reliably mistakes:
|
|
132
|
+
* timestamps ("57:22.283Z]"), ordinary words ("Sozialversicherungsnummer") and
|
|
133
|
+
* region names ("us-east-1"). All three sit after a colon or equals sign, which
|
|
134
|
+
* is all that rule asks for, and all three clear the entropy threshold.
|
|
135
|
+
*
|
|
136
|
+
* The word test is narrow on purpose: purely alphabetic *and* shaped like one
|
|
137
|
+
* natural word. A random all-letter token would be mixed case and is left alone
|
|
138
|
+
* — losing a real secret is the expensive direction.
|
|
139
|
+
*/
|
|
140
|
+
function suppressSecretNotASecret(_text, match) {
|
|
141
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.SECRET)
|
|
142
|
+
return false;
|
|
143
|
+
const bare = coreToken(match.text);
|
|
144
|
+
if (TIMESTAMP_FRAGMENT.test(bare))
|
|
145
|
+
return true;
|
|
146
|
+
if (NOT_A_SECRET.test(bare))
|
|
147
|
+
return true;
|
|
148
|
+
const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
|
|
149
|
+
const oneWord = bare === bare.toLowerCase() ||
|
|
150
|
+
(bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
|
|
151
|
+
bare.slice(1) === bare.slice(1).toLowerCase());
|
|
152
|
+
return alpha && oneWord;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* A money amount followed by its ISO 4217 code is not a registration. Spain's
|
|
156
|
+
* plate shape is four digits then three consonants, and the Nordic and Central
|
|
157
|
+
* European currency codes are all consonants: "2297 DKK" read as a Spanish plate
|
|
158
|
+
* 487 times across the corpus.
|
|
159
|
+
*/
|
|
160
|
+
function suppressPlateAsCurrencyAmount(text, match) {
|
|
161
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.LICENSE_PLATE)
|
|
162
|
+
return false;
|
|
163
|
+
if (CURRENCY_CODE_TAIL.test(match.text))
|
|
164
|
+
return true;
|
|
165
|
+
return CURRENCY_CODE_AFTER.test(text.slice(match.end, match.end + 12));
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* A dotted quad is an address, not a tax number. Germany's tax-number shape
|
|
169
|
+
* allows dots between its digit groups, which makes it a superset of IPv4. The
|
|
170
|
+
* address is still redacted — the IP_ADDRESS rule claims the same span — so this
|
|
171
|
+
* only corrects the label.
|
|
172
|
+
*/
|
|
173
|
+
function suppressTaxidAsIpAddress(_text, match) {
|
|
174
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.TAX_ID)
|
|
175
|
+
return false;
|
|
176
|
+
return DOTTED_QUAD.test(match.text.trim());
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Reject two phone shapes that are arithmetic rather than contact details: a
|
|
180
|
+
* year range ("Schooljaar 2025-2026") and the fractional part of a decimal
|
|
181
|
+
* amount ("0.034865 BTC"). Both clear every phone pattern's shape test.
|
|
182
|
+
*/
|
|
183
|
+
function suppressPhoneAsNumberRange(text, match) {
|
|
184
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
|
|
185
|
+
return false;
|
|
186
|
+
if (YEAR_RANGE.test(match.text.trim()))
|
|
187
|
+
return true;
|
|
188
|
+
return DECIMAL_TAIL_BEFORE.test(text.slice(Math.max(0, match.start - 2), match.start));
|
|
189
|
+
}
|
|
37
190
|
function suppressCurrency(text, match) {
|
|
38
191
|
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
39
192
|
return false;
|
|
@@ -90,6 +243,16 @@ function suppressYearAsPostal(text, match) {
|
|
|
90
243
|
const clean = match.text.trim();
|
|
91
244
|
if (!RECENT_YEAR.test(clean))
|
|
92
245
|
return false;
|
|
246
|
+
// A date construction *touching* the candidate settles it, whatever the rest
|
|
247
|
+
// of the document says. Without this, the postal-context test below rescued
|
|
248
|
+
// every year in every document with an address in it — and "Adresse", "rue"
|
|
249
|
+
// and "Str." appear in the header of essentially every business letter.
|
|
250
|
+
// Measured: 1,691 of 3,322 postal false positives were plausible years,
|
|
251
|
+
// 1,636 of them literally "2025".
|
|
252
|
+
if (DATE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start)))
|
|
253
|
+
return true;
|
|
254
|
+
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
255
|
+
return true;
|
|
93
256
|
const [before, after] = getContext(text, match.start, match.end);
|
|
94
257
|
const context = before + after;
|
|
95
258
|
// Keep as postal code if postal/address context nearby
|
|
@@ -453,22 +616,59 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
453
616
|
return true;
|
|
454
617
|
return false;
|
|
455
618
|
}
|
|
619
|
+
/**
|
|
620
|
+
* A delimited data row: at least three fields separated by one delimiter.
|
|
621
|
+
* Export formats carry their meaning in the column, not in a nearby word, so a
|
|
622
|
+
* value filling an entire field has structural context even when the line
|
|
623
|
+
* contains no cue.
|
|
624
|
+
*/
|
|
625
|
+
const DELIMITED_ROW = /^[^\n]*?([,;|\t])[^\n]*?\1[^\n]*?\1/;
|
|
626
|
+
/**
|
|
627
|
+
* Types for which filling a delimited field counts as context. Narrow shapes
|
|
628
|
+
* only: applied to every type it was measured a net loss — SECRET gained 1,678
|
|
629
|
+
* false positives, CHAMBER_OF_COMMERCE 456 and POSTAL_CODE 295. A broad shape
|
|
630
|
+
* plus a required cue is a deliberate pairing, and removing the cue leaves only
|
|
631
|
+
* the broad shape.
|
|
632
|
+
*/
|
|
633
|
+
const DELIMITED_FIELD_TYPES = new Set([
|
|
634
|
+
types_js_1.EntityType.PHONE, types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH,
|
|
635
|
+
]);
|
|
636
|
+
/** True when the span is exactly one field of a delimited row. */
|
|
637
|
+
function fillsADelimitedField(text, start, end) {
|
|
638
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
639
|
+
let lineEnd = text.indexOf("\n", end);
|
|
640
|
+
if (lineEnd < 0)
|
|
641
|
+
lineEnd = text.length;
|
|
642
|
+
if (!DELIMITED_ROW.test(text.slice(lineStart, lineEnd)))
|
|
643
|
+
return false;
|
|
644
|
+
const before = text.slice(lineStart, start);
|
|
645
|
+
const after = text.slice(end, lineEnd);
|
|
646
|
+
const opens = before === "" || ",;|\t".includes(before[before.length - 1]);
|
|
647
|
+
const closes = after === "" || ",;|\t".includes(after[0]);
|
|
648
|
+
return opens && closes;
|
|
649
|
+
}
|
|
456
650
|
function suppressRequiresContext(text, match) {
|
|
457
651
|
if (!match.patternDef.requiresContext || match.patternDef.contextKeywords.length === 0)
|
|
458
652
|
return false;
|
|
653
|
+
// A value filling an entire field of a delimited row is introduced by its
|
|
654
|
+
// column, not by a word, so that is context — but only for narrow shapes.
|
|
655
|
+
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
656
|
+
fillsADelimitedField(text, match.start, match.end))
|
|
657
|
+
return false;
|
|
459
658
|
const [before, after] = getContext(text, match.start, match.end);
|
|
460
659
|
const context = (before + " " + after).toLowerCase();
|
|
461
660
|
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
462
661
|
}
|
|
463
662
|
const TYPE_SUPPRESSORS = {
|
|
464
|
-
[types_js_1.EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap],
|
|
663
|
+
[types_js_1.EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
465
664
|
[types_js_1.EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
466
665
|
[types_js_1.EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
467
|
-
[types_js_1.EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
666
|
+
[types_js_1.EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
468
667
|
[types_js_1.EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier],
|
|
469
668
|
[types_js_1.EntityType.BIC]: [suppressBicWithoutEvidence],
|
|
470
669
|
[types_js_1.EntityType.BANK_ACCOUNT]: [suppressReference],
|
|
471
|
-
[types_js_1.EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict],
|
|
670
|
+
[types_js_1.EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
671
|
+
[types_js_1.EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
472
672
|
[types_js_1.EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
473
673
|
};
|
|
474
674
|
function shouldSuppress(text, match) {
|
|
@@ -191,7 +191,7 @@ const BE = {
|
|
|
191
191
|
p(EntityType.NATIONAL_ID, String.raw `\b\d{2}\.?\d{2}\.?\d{2}[\-.]?\d{3}\.?\d{2}\b`, "belgian_nn"),
|
|
192
192
|
p(EntityType.IBAN, String.raw `\bBE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\b`, "iban"),
|
|
193
193
|
p(EntityType.VAT, String.raw `\bBE\s?0\d{3}\.?\d{3}\.?\d{3}\b`, "belgian_vat"),
|
|
194
|
-
p(EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer"], true),
|
|
194
|
+
p(EntityType.CHAMBER_OF_COMMERCE, String.raw `\b0\d{3}\.?\d{3}\.?\d{3}\b`, null, "", ["KBO", "BCE", "ondernemingsnummer", "numéro d'entreprise", "enterprise number", "bedrijfsnummer", "Kruispuntbank", "Banque-Carrefour", "Banque Carrefour", "Ondernemingen", "Entreprises", "onder nummer"], true),
|
|
195
195
|
p(EntityType.PHONE, String.raw `\b0[1-9]\d{0,2}[/\s.\-]?\d{2,3}[.\s\-]?\d{2,3}[.\s\-]?\d{2,3}\b`),
|
|
196
196
|
p(EntityType.PHONE, String.raw `\+32\s?\d{1,3}[\s.\-]?\d{2,3}[\s.\-]?\d{2}[\s.\-]?\d{2}`),
|
|
197
197
|
p(EntityType.PASSPORT, String.raw `\b[A-Z]{2}\d{6}\b`, null, "", ["paspoort", "passport", "passeport", "reisdocument", "travel document", "document de voyage"], true),
|
|
@@ -223,7 +223,7 @@ const DE = {
|
|
|
223
223
|
p(EntityType.LICENSE_PLATE, String.raw `\b[A-ZÄÖÜ]{1,3}[\s\-][A-Z]{1,2}[\s\-]?\d{1,4}[EH]?\b`),
|
|
224
224
|
p(EntityType.POSTAL_CODE, String.raw `\b\d{5}\b`, null, "", ["PLZ", "Postleitzahl", "postal code", "postcode", "Anschrift", "Adresse", "Wohnort", "Ort", "Stadt", "wohnt", "wohnhaft", "Straße", "Str.", "Weg", "Platz", "Allee", "Ring"], true),
|
|
225
225
|
p(EntityType.CHAMBER_OF_COMMERCE, String.raw `\bHR[AB]\s?\d{4,6}\b`),
|
|
226
|
-
p(EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer"], true),
|
|
226
|
+
p(EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer", "SVNR", "SV-Nr", "RVNR"], true),
|
|
227
227
|
p(EntityType.TAX_ID, String.raw `\b\d{2,3}/\d{3,4}/\d{4,5}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
228
228
|
p(EntityType.TAX_ID, String.raw `\b\d{13}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
229
229
|
p(EntityType.HEALTH_INSURANCE, String.raw `\b[A-Z]\d{9}\b`, null, "", ["Versichertennummer", "Krankenversicherung", "KV-Nummer", "Krankenkasse", "GKV", "Versichertenkarte", "Versicherter", "Versicherte", "KVNR", "Krankenversichertennummer"], true),
|
|
@@ -384,8 +384,15 @@ const ES = {
|
|
|
384
384
|
p(EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}\b`),
|
|
385
385
|
p(EntityType.PHONE, String.raw `\b[679]\d{8}\b`),
|
|
386
386
|
p(EntityType.PHONE, String.raw `\b[679]\d[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
387
|
+
// 3-2-2-2. Just as common as the groupings above and absent until now:
|
|
388
|
+
// "705 97 55 11" matched no phone pattern at all, which cost 674 Spanish
|
|
389
|
+
// numbers — the single largest PHONE gap.
|
|
390
|
+
p(EntityType.PHONE, String.raw `\b[679]\d{2}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}\b`),
|
|
387
391
|
p(EntityType.PHONE, String.raw `\+34\s?[679]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}`),
|
|
388
|
-
|
|
392
|
+
// A single space, not \s: \s matched a line break, so a year ending one
|
|
393
|
+
// line and a label starting the next ("2002\nCPR") read as one
|
|
394
|
+
// registration — 310 times across the corpus.
|
|
395
|
+
p(EntityType.LICENSE_PLATE, String.raw `\b\d{4}[ ]?[BCDFGHJKLMNPRSTVWXYZ]{3}\b`),
|
|
389
396
|
p(EntityType.POSTAL_CODE, String.raw `\b(?:0[1-9]|[1-4]\d|5[0-2])\d{3}\b`, null, "", ["código postal", "C.P.", "CP", "dirección", "domicilio", "calle", "avenida", "plaza", "paseo", "camino", "Postal:"], true),
|
|
390
397
|
],
|
|
391
398
|
};
|
package/dist/esm/rules/engine.js
CHANGED
|
@@ -53,6 +53,46 @@ function warnUnknownCountry(code) {
|
|
|
53
53
|
`shared country-independent patterns only (email, IBAN, international ` +
|
|
54
54
|
`phone, credit card, ...). Available: ${available.join(", ")}`);
|
|
55
55
|
}
|
|
56
|
+
// ── Local cues: what the document calls the value next to it ────────────
|
|
57
|
+
//
|
|
58
|
+
// Country evidence resolves which *national scheme* owns an ambiguous value, but
|
|
59
|
+
// it never looks at the label sitting immediately in front of it. So
|
|
60
|
+
// "Phone: 0705237535" was reported as a Swedish national ID: the digits pass
|
|
61
|
+
// that checksum, the document is Swedish, and nothing consulted the word
|
|
62
|
+
// "Phone".
|
|
63
|
+
//
|
|
64
|
+
// A cue touching the span is the strongest statement the document makes about
|
|
65
|
+
// what the value *is*, so it outranks the checksum — but only among candidates
|
|
66
|
+
// covering the **same span**, so it decides the label and can never change which
|
|
67
|
+
// characters are masked. That is what keeps invariant I1 intact.
|
|
68
|
+
//
|
|
69
|
+
// Measured: 263 phone numbers and 18 French SIREN were reported as national IDs,
|
|
70
|
+
// each with an explicit cue in front of it.
|
|
71
|
+
const LOCAL_CUES = [
|
|
72
|
+
[EntityType.PHONE,
|
|
73
|
+
/(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell)\w*\s*[:.\-]?\s*$/i],
|
|
74
|
+
[EntityType.CHAMBER_OF_COMMERCE,
|
|
75
|
+
/(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr)\w*\s*[:.\-]?\s*$/i],
|
|
76
|
+
[EntityType.VAT,
|
|
77
|
+
/(?:vat|btw|tva|ust|iva|moms|alv|mwst)[\w.\-]*\s*[:.\-]?\s*$/i],
|
|
78
|
+
[EntityType.BIC,
|
|
79
|
+
/(?:bic|swift)[\w/]*\s*[:.\-]?\s*\(?\s*$/i],
|
|
80
|
+
[EntityType.NATIONAL_ID,
|
|
81
|
+
/(?:bsn|personnummer|rijksregisternummer|national\s*id|identiteitsnummer|henkilötunnus|kennitala|cpr|nif|dni)\w*\s*[:.\-]?\s*$/i],
|
|
82
|
+
];
|
|
83
|
+
// How far back a cue may sit. Short on purpose: a cue is only evidence about the
|
|
84
|
+
// value it introduces, which is the lesson of the postal-code year defect, where
|
|
85
|
+
// a cue 150 characters away licensed every number in the window.
|
|
86
|
+
const CUE_WINDOW = 22;
|
|
87
|
+
/** 1 when a cue naming `entityType` sits immediately before the span. */
|
|
88
|
+
function localCueBonus(text, start, entityType) {
|
|
89
|
+
const before = text.slice(Math.max(0, start - CUE_WINDOW), start);
|
|
90
|
+
for (const [cuedType, pattern] of LOCAL_CUES) {
|
|
91
|
+
if (cuedType === entityType && pattern.test(before))
|
|
92
|
+
return 1;
|
|
93
|
+
}
|
|
94
|
+
return 0;
|
|
95
|
+
}
|
|
56
96
|
export class RuleEngine {
|
|
57
97
|
matcher = new MultiPatternMatcher();
|
|
58
98
|
loadedCountries = new Set();
|
|
@@ -336,6 +376,7 @@ export class RuleEngine {
|
|
|
336
376
|
? match.countryCode : null;
|
|
337
377
|
candidates.push({
|
|
338
378
|
blindPriority,
|
|
379
|
+
cue: localCueBonus(text, match.start, match.patternDef.entityType),
|
|
339
380
|
priority,
|
|
340
381
|
length: match.end - match.start,
|
|
341
382
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
@@ -353,7 +394,7 @@ export class RuleEngine {
|
|
|
353
394
|
});
|
|
354
395
|
}
|
|
355
396
|
for (const d of detectStructuralDob(text)) {
|
|
356
|
-
candidates.push({ blindPriority: 1, priority: 1, length: d.end - d.start,
|
|
397
|
+
candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
|
|
357
398
|
countryScore: 0, inScope: 1, det: d });
|
|
358
399
|
}
|
|
359
400
|
const detections = this.deduplicate(candidates);
|
|
@@ -416,9 +457,10 @@ export class RuleEngine {
|
|
|
416
457
|
const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first } country-blind
|
|
417
458
|
tierOf(b) - tierOf(a) || // best tier here } — decides
|
|
418
459
|
a.det.start - b.det.start || // leftmost } WHICH span
|
|
419
|
-
b.
|
|
420
|
-
b.
|
|
421
|
-
b.
|
|
460
|
+
b.cue - a.cue || // local cue } country-aware
|
|
461
|
+
b.priority - a.priority || // country priority } — decides the
|
|
462
|
+
b.countryScore - a.countryScore || // country evidence } LABEL only
|
|
463
|
+
b.inScope - a.inScope); // declared scope }
|
|
422
464
|
const result = [];
|
|
423
465
|
const occupied = new Set();
|
|
424
466
|
for (const { det } of sorted) {
|
|
@@ -16,7 +16,8 @@ const AMOUNT_LABEL_BEFORE = /(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|T
|
|
|
16
16
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
17
17
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
18
18
|
const UNIT_AFTER = /^\s*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
19
|
-
|
|
19
|
+
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
20
|
+
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.]?\s*$/i;
|
|
20
21
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
21
22
|
const MATH_BEFORE = /[=+\-×÷*/]\s*$/;
|
|
22
23
|
const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
@@ -25,12 +26,164 @@ const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
|
25
26
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
26
27
|
const SEQUENTIAL_PATTERNS = /^(?:0{6,}|1234567890?|0123456789|9876543210?|1111111111?|000000000|123456789)$/;
|
|
27
28
|
// YEAR_PATTERN kept for reference but replaced by RECENT_YEAR in suppressYearAsPostal
|
|
28
|
-
|
|
29
|
+
// The leading \b is load-bearing. Without it the short alternatives match the
|
|
30
|
+
// *tail* of an ordinary word: "NIS" matched "tālru**nis** ", the Latvian for
|
|
31
|
+
// "telephone", so every Latvian phone number was suppressed as though it sat
|
|
32
|
+
// behind a Belgian national-ID label. 653 misses from a missing word boundary.
|
|
33
|
+
const ID_LABEL_BEFORE = /\b(?:BSN|RR|NN|NIR|INSZ|NISS|NIS|Steuer-?ID|TIN|NIF|NIE|SSN|rijksregisternummer|numéro\s*national|national\s*number|matricule|Ausweisnummer|Personalausweis|Versichertennummer|KVNR|KV-Nr|Steuernummer|St\.\-Nr|StNr|Finanzamt\s+ist|Ondernemingen\s+onder\s+nummer|ondernemingsnummer|numéro\s*d'entreprise|enterprise\s*number|Kruispuntbank)\s*[:.]?\s*$/i;
|
|
29
34
|
const SERVICE_NUMBER = /^0800[\-\s]/;
|
|
30
35
|
const DATE_PATTERN_FULL = /^\d{2}[-/.]\d{2}[-/.]\d{4}$|^\d{4}[-/.]\d{2}[-/.]\d{2}$/;
|
|
31
36
|
const PASSPORT_CONTEXT_BEFORE = /(?:Reisepass|passport|passeport|paspoort|Bisheriger\s+Reisepass)\s*(?:Nr\.?|Nummer|nummer|number|n[°o])?\s*[:.]?\s*$/i;
|
|
32
37
|
const SE_ORG_CONTEXT_BEFORE = /(?:org\.?\s*nr\.?|organisationsnummer|organisationsnr|Bolagsverket|företag)\s*[:.]?\s*$/i;
|
|
33
38
|
const NUMERIC_TYPES = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID, EntityType.POSTAL_CODE]);
|
|
39
|
+
// ── Date adjacency, for the postal-code year gate ───────────────────────
|
|
40
|
+
//
|
|
41
|
+
// Month names in the languages this engine covers. Long, but it is data: the
|
|
42
|
+
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
43
|
+
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
44
|
+
const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
45
|
+
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
46
|
+
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
47
|
+
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
48
|
+
"|januari|februari|maart|mei|augustus|mrt" +
|
|
49
|
+
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
50
|
+
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
51
|
+
"|janeiro|fevereiro|março|marco|maio|junho|julho|setembro|outubro|novembro|dezembro" +
|
|
52
|
+
"|marts|augusti" +
|
|
53
|
+
"|tammikuu|helmikuu|maaliskuu|huhtikuu|toukokuu|kesäkuu|heinäkuu|elokuu|syyskuu|lokakuu|marraskuu|joulukuu" +
|
|
54
|
+
"|jaanuar|veebruar|aprill|juuni|juuli|oktoober|detsember" +
|
|
55
|
+
"|janvāris|februāris|aprīlis|maijs|jūnijs|jūlijs|augusts|septembris|oktobris|novembris|decembris" +
|
|
56
|
+
"|stycznia|lutego|marca|kwietnia|maja|czerwca|lipca|sierpnia|września|października|listopada|grudnia" +
|
|
57
|
+
"|ledna|února|března|dubna|května|června|července|srpna|září|října|listopadu|prosince" +
|
|
58
|
+
"|marec|junij|julij|avgust|siječnja|veljače|ožujka|travnja|svibnja|lipnja|srpnja|kolovoza|rujna|studenoga|prosinca" +
|
|
59
|
+
"|március|április|május|június|július|augusztus|szeptember|október" +
|
|
60
|
+
"|ianuarie|februarie|martie|aprilie|iunie|iulie|septembrie|octombrie|noiembrie|decembrie" +
|
|
61
|
+
"|ιανουαρίου|φεβρουαρίου|μαρτίου|απριλίου|μαΐου|ιουνίου|ιουλίου|αυγούστου|σεπτεμβρίου|οκτωβρίου|νοεμβρίου|δεκεμβρίου" +
|
|
62
|
+
"|януари|февруари|март|април|май|юни|юли|август|септември|октомври|ноември|декември";
|
|
63
|
+
/** "since" cues, which introduce a bare year in every one of these languages. */
|
|
64
|
+
const SINCE_CUES = "seit|since|depuis|sinds|sedert|vanaf|desde|dal|od|sedan|siden|alates|alkaen|από|от|din|iz|ab";
|
|
65
|
+
/**
|
|
66
|
+
* A month name, a "since" cue, or a numeric date tail immediately before the
|
|
67
|
+
* candidate. Adjacent-only on purpose — a cue 150 characters away is what made
|
|
68
|
+
* this a bug in the first place.
|
|
69
|
+
*/
|
|
70
|
+
const DATE_BEFORE = new RegExp(`(?:\\b(?:${MONTH_NAMES})\\b\\.?|\\b(?:${SINCE_CUES})\\b|\\d{1,2}[./-]\\d{1,2}[./-])\\s*$`, "iu");
|
|
71
|
+
/** A date separator immediately after, making the candidate a leading year. */
|
|
72
|
+
const DATE_AFTER = /^[./-]\d{1,2}[./-]\d{1,2}\b/;
|
|
73
|
+
/**
|
|
74
|
+
* ISO 4217 codes made only of the consonants a Spanish plate accepts, so a money
|
|
75
|
+
* amount reads as a registration. This is why "2297 DKK" was a plate.
|
|
76
|
+
*/
|
|
77
|
+
const CURRENCY_CODE = "(?:DKK|SEK|NOK|ISK|CZK|PLN|HUF|RON|BGN|HRK|CHF|GBP|TRY|RSD|MKD|BYN|KZT|CNY|JPY|KRW|ZAR|BRL|MXN|CLP|COP|PLZ|SKK|TRL)";
|
|
78
|
+
const CURRENCY_CODE_TAIL = new RegExp(`\\s${CURRENCY_CODE}$`);
|
|
79
|
+
const CURRENCY_CODE_AFTER = new RegExp(`^\\s*${CURRENCY_CODE}\\b`);
|
|
80
|
+
/** Log-timestamp tails, which the ":"-anchored secret rule picks up. */
|
|
81
|
+
const TIMESTAMP_FRAGMENT = /^\d{1,2}:\d{2}(?::\d{2})?(?:[.,]\d{1,6})?Z?$/;
|
|
82
|
+
/** Cloud region names and similar well-known configuration values. */
|
|
83
|
+
const NOT_A_SECRET = new RegExp("^(?:af|ap|ca|cn|eu|il|me|sa|us|gov)-(?:north|south|east|west|central" +
|
|
84
|
+
"|northeast|northwest|southeast|southwest)-\\d[a-z]?$" +
|
|
85
|
+
"|^(?:application|text|image|audio|video|multipart)/[\\w.+-]+$" +
|
|
86
|
+
"|^(?:utf|iso|windows)-[\\d-]+$" +
|
|
87
|
+
"|^(?:no-cache|no-store|max-age|gzip|deflate|identity|chunked)\\b", "i");
|
|
88
|
+
const EXACT_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
|
|
89
|
+
const BIC_SHAPE = /^[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?$/;
|
|
90
|
+
const BIC_CUE_BEFORE = /(?:BIC|SWIFT|BIC\/SWIFT)\s*[:=]?\s*\(?\s*$/i;
|
|
91
|
+
/** A dotted quad with every octet in range. */
|
|
92
|
+
const DOTTED_QUAD = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
|
|
93
|
+
/** Two plausible years joined by a hyphen — a school year, a contract term. */
|
|
94
|
+
const YEAR_RANGE = /^(19|20)\d{2}\s?[-–/]\s?(19|20)\d{2}$/;
|
|
95
|
+
/** A digit and a decimal point immediately before, so the match is a fraction. */
|
|
96
|
+
const DECIMAL_TAIL_BEFORE = /\d[.,]$/;
|
|
97
|
+
/**
|
|
98
|
+
* Punctuation the greedy "[^\s]{8,}" secret rule sweeps up with the token: it
|
|
99
|
+
* matched "TRIONL2U)." out of "(BIC: TRIONL2U).". Stripped only for the shape
|
|
100
|
+
* tests — the emitted span is never altered here.
|
|
101
|
+
*/
|
|
102
|
+
const PUNCT_EDGES = /^[\s([{<"'`]+|[\s)\]}>"'`.,;:!?]+$/g;
|
|
103
|
+
function coreToken(raw) {
|
|
104
|
+
return raw.replace(PUNCT_EDGES, "").replace(PUNCT_EDGES, "");
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* A generic secret must not claim a span that is a specific known type.
|
|
108
|
+
*
|
|
109
|
+
* The high-entropy rules are deliberately broad and they carry a validator, so
|
|
110
|
+
* they reach the top priority tier while the structured detector for the same
|
|
111
|
+
* characters sits at the bottom with nothing to offer. Measured on 152,300
|
|
112
|
+
* documents: 687 UUIDs and 140 BICs were reported as SECRET, each counted twice
|
|
113
|
+
* over — a false positive for SECRET and a miss for the type that should have
|
|
114
|
+
* had it.
|
|
115
|
+
*/
|
|
116
|
+
function suppressSecretOverStructured(text, match) {
|
|
117
|
+
if (match.patternDef.entityType !== EntityType.SECRET)
|
|
118
|
+
return false;
|
|
119
|
+
const token = coreToken(match.text);
|
|
120
|
+
if (EXACT_UUID.test(token))
|
|
121
|
+
return true;
|
|
122
|
+
if (BIC_SHAPE.test(token)) {
|
|
123
|
+
return BIC_CUE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
|
|
124
|
+
}
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Reject the three things the ":"-anchored secret rule reliably mistakes:
|
|
129
|
+
* timestamps ("57:22.283Z]"), ordinary words ("Sozialversicherungsnummer") and
|
|
130
|
+
* region names ("us-east-1"). All three sit after a colon or equals sign, which
|
|
131
|
+
* is all that rule asks for, and all three clear the entropy threshold.
|
|
132
|
+
*
|
|
133
|
+
* The word test is narrow on purpose: purely alphabetic *and* shaped like one
|
|
134
|
+
* natural word. A random all-letter token would be mixed case and is left alone
|
|
135
|
+
* — losing a real secret is the expensive direction.
|
|
136
|
+
*/
|
|
137
|
+
function suppressSecretNotASecret(_text, match) {
|
|
138
|
+
if (match.patternDef.entityType !== EntityType.SECRET)
|
|
139
|
+
return false;
|
|
140
|
+
const bare = coreToken(match.text);
|
|
141
|
+
if (TIMESTAMP_FRAGMENT.test(bare))
|
|
142
|
+
return true;
|
|
143
|
+
if (NOT_A_SECRET.test(bare))
|
|
144
|
+
return true;
|
|
145
|
+
const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
|
|
146
|
+
const oneWord = bare === bare.toLowerCase() ||
|
|
147
|
+
(bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
|
|
148
|
+
bare.slice(1) === bare.slice(1).toLowerCase());
|
|
149
|
+
return alpha && oneWord;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* A money amount followed by its ISO 4217 code is not a registration. Spain's
|
|
153
|
+
* plate shape is four digits then three consonants, and the Nordic and Central
|
|
154
|
+
* European currency codes are all consonants: "2297 DKK" read as a Spanish plate
|
|
155
|
+
* 487 times across the corpus.
|
|
156
|
+
*/
|
|
157
|
+
function suppressPlateAsCurrencyAmount(text, match) {
|
|
158
|
+
if (match.patternDef.entityType !== EntityType.LICENSE_PLATE)
|
|
159
|
+
return false;
|
|
160
|
+
if (CURRENCY_CODE_TAIL.test(match.text))
|
|
161
|
+
return true;
|
|
162
|
+
return CURRENCY_CODE_AFTER.test(text.slice(match.end, match.end + 12));
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* A dotted quad is an address, not a tax number. Germany's tax-number shape
|
|
166
|
+
* allows dots between its digit groups, which makes it a superset of IPv4. The
|
|
167
|
+
* address is still redacted — the IP_ADDRESS rule claims the same span — so this
|
|
168
|
+
* only corrects the label.
|
|
169
|
+
*/
|
|
170
|
+
function suppressTaxidAsIpAddress(_text, match) {
|
|
171
|
+
if (match.patternDef.entityType !== EntityType.TAX_ID)
|
|
172
|
+
return false;
|
|
173
|
+
return DOTTED_QUAD.test(match.text.trim());
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Reject two phone shapes that are arithmetic rather than contact details: a
|
|
177
|
+
* year range ("Schooljaar 2025-2026") and the fractional part of a decimal
|
|
178
|
+
* amount ("0.034865 BTC"). Both clear every phone pattern's shape test.
|
|
179
|
+
*/
|
|
180
|
+
function suppressPhoneAsNumberRange(text, match) {
|
|
181
|
+
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
182
|
+
return false;
|
|
183
|
+
if (YEAR_RANGE.test(match.text.trim()))
|
|
184
|
+
return true;
|
|
185
|
+
return DECIMAL_TAIL_BEFORE.test(text.slice(Math.max(0, match.start - 2), match.start));
|
|
186
|
+
}
|
|
34
187
|
function suppressCurrency(text, match) {
|
|
35
188
|
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
36
189
|
return false;
|
|
@@ -87,6 +240,16 @@ function suppressYearAsPostal(text, match) {
|
|
|
87
240
|
const clean = match.text.trim();
|
|
88
241
|
if (!RECENT_YEAR.test(clean))
|
|
89
242
|
return false;
|
|
243
|
+
// A date construction *touching* the candidate settles it, whatever the rest
|
|
244
|
+
// of the document says. Without this, the postal-context test below rescued
|
|
245
|
+
// every year in every document with an address in it — and "Adresse", "rue"
|
|
246
|
+
// and "Str." appear in the header of essentially every business letter.
|
|
247
|
+
// Measured: 1,691 of 3,322 postal false positives were plausible years,
|
|
248
|
+
// 1,636 of them literally "2025".
|
|
249
|
+
if (DATE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start)))
|
|
250
|
+
return true;
|
|
251
|
+
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
252
|
+
return true;
|
|
90
253
|
const [before, after] = getContext(text, match.start, match.end);
|
|
91
254
|
const context = before + after;
|
|
92
255
|
// Keep as postal code if postal/address context nearby
|
|
@@ -450,22 +613,59 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
450
613
|
return true;
|
|
451
614
|
return false;
|
|
452
615
|
}
|
|
616
|
+
/**
|
|
617
|
+
* A delimited data row: at least three fields separated by one delimiter.
|
|
618
|
+
* Export formats carry their meaning in the column, not in a nearby word, so a
|
|
619
|
+
* value filling an entire field has structural context even when the line
|
|
620
|
+
* contains no cue.
|
|
621
|
+
*/
|
|
622
|
+
const DELIMITED_ROW = /^[^\n]*?([,;|\t])[^\n]*?\1[^\n]*?\1/;
|
|
623
|
+
/**
|
|
624
|
+
* Types for which filling a delimited field counts as context. Narrow shapes
|
|
625
|
+
* only: applied to every type it was measured a net loss — SECRET gained 1,678
|
|
626
|
+
* false positives, CHAMBER_OF_COMMERCE 456 and POSTAL_CODE 295. A broad shape
|
|
627
|
+
* plus a required cue is a deliberate pairing, and removing the cue leaves only
|
|
628
|
+
* the broad shape.
|
|
629
|
+
*/
|
|
630
|
+
const DELIMITED_FIELD_TYPES = new Set([
|
|
631
|
+
EntityType.PHONE, EntityType.DOB, EntityType.DATE_OF_DEATH,
|
|
632
|
+
]);
|
|
633
|
+
/** True when the span is exactly one field of a delimited row. */
|
|
634
|
+
function fillsADelimitedField(text, start, end) {
|
|
635
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
636
|
+
let lineEnd = text.indexOf("\n", end);
|
|
637
|
+
if (lineEnd < 0)
|
|
638
|
+
lineEnd = text.length;
|
|
639
|
+
if (!DELIMITED_ROW.test(text.slice(lineStart, lineEnd)))
|
|
640
|
+
return false;
|
|
641
|
+
const before = text.slice(lineStart, start);
|
|
642
|
+
const after = text.slice(end, lineEnd);
|
|
643
|
+
const opens = before === "" || ",;|\t".includes(before[before.length - 1]);
|
|
644
|
+
const closes = after === "" || ",;|\t".includes(after[0]);
|
|
645
|
+
return opens && closes;
|
|
646
|
+
}
|
|
453
647
|
function suppressRequiresContext(text, match) {
|
|
454
648
|
if (!match.patternDef.requiresContext || match.patternDef.contextKeywords.length === 0)
|
|
455
649
|
return false;
|
|
650
|
+
// A value filling an entire field of a delimited row is introduced by its
|
|
651
|
+
// column, not by a word, so that is context — but only for narrow shapes.
|
|
652
|
+
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
653
|
+
fillsADelimitedField(text, match.start, match.end))
|
|
654
|
+
return false;
|
|
456
655
|
const [before, after] = getContext(text, match.start, match.end);
|
|
457
656
|
const context = (before + " " + after).toLowerCase();
|
|
458
657
|
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
459
658
|
}
|
|
460
659
|
const TYPE_SUPPRESSORS = {
|
|
461
|
-
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap],
|
|
660
|
+
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
462
661
|
[EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
463
662
|
[EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
464
|
-
[EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
663
|
+
[EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
465
664
|
[EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier],
|
|
466
665
|
[EntityType.BIC]: [suppressBicWithoutEvidence],
|
|
467
666
|
[EntityType.BANK_ACCOUNT]: [suppressReference],
|
|
468
|
-
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict],
|
|
667
|
+
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
668
|
+
[EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
469
669
|
[EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
470
670
|
};
|
|
471
671
|
export function shouldSuppress(text, match) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.5",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|