euredact 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +403 -39
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +124 -14
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +63 -10
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +292 -22
- package/dist/cjs/rules/validators.js +50 -0
- package/dist/cjs/sdk.js +18 -7
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +83 -1
- package/dist/esm/cloud/client.js +119 -22
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +63 -10
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +291 -21
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +47 -0
- package/dist/esm/sdk.d.ts +51 -4
- package/dist/esm/sdk.js +18 -9
- package/dist/esm/types.d.ts +58 -2
- package/dist/esm/types.js +4 -1
- package/package.json +12 -4
package/dist/esm/rules/cues.js
CHANGED
|
@@ -90,7 +90,7 @@ export const CUES = [
|
|
|
90
90
|
/(?<![A-Za-z0-9_])(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell|tālrunis|τηλέφωνο|τηλ)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
91
91
|
// National identity number.
|
|
92
92
|
[EntityType.NATIONAL_ID,
|
|
93
|
-
/(?<![A-Za-z0-9_])(?:bsn|burgerservicenummer(?:\s*\(bsn\))?|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|
|
|
93
|
+
/(?<![A-Za-z0-9_])(?:bsn(?!r)|burgerservicenummer(?:\s*\(bsn\))?|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|dnie?(?![A-Za-z]|[^\x00-\x7F])|rr|nn|nir|insz|niss|nis|matricule|id\s*card(?:\s*(?:no|nr|number))?|ausweisnummer|personalausweis|osobní\s*číslo|ЕГН|EGN|dow[óo]d(?:u|em)?\s*osobist(?:y|ego|ym)|nr\s*dowodu|ahv(?:-?(?:nr|nummer))?|avs)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
94
94
|
// Cyprus: "ID number" in both official languages, plus the Greek abbreviation
|
|
95
95
|
// ΑΔΤ (Αστυνομική Ταυτότητα). Neither has a phone reading.
|
|
96
96
|
[EntityType.NATIONAL_ID,
|
|
@@ -111,16 +111,16 @@ export const CUES = [
|
|
|
111
111
|
/(?<![A-Za-z0-9_])(?:αφμ|α\.φ\.μ\.|steuerliche\s*identifikations-?nummer|steuer-?identifikations-?nummer|steuer-?id-?nr\.?|steuer-?id|steuernummer|st\.?-?nr|stnr|tin|finanzamt\s+ist|tax\s*(?:id|no|number)|daňové\s*číslo|numer\s*podatkowy)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
112
112
|
// Health insurance: the insured person.
|
|
113
113
|
[EntityType.HEALTH_INSURANCE,
|
|
114
|
-
/(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer|medical\s*card(?:\s*(?:no|number))?)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
114
|
+
/(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer|medical\s*card(?:\s*(?:no|number))?|nhs(?:\s*(?:no|number))?)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
115
115
|
// Healthcare provider: the clinician or practice, not the patient — the Dutch
|
|
116
116
|
// AGB-code, the German LANR, the UK GMC number.
|
|
117
117
|
[EntityType.HEALTHCARE_PROVIDER,
|
|
118
|
-
/(?<![A-Za-z0-9_])(?:agb-?code|lanr|bsnr|gmc(?:\s*(?:no|number|reg))?|big
|
|
118
|
+
/(?<![A-Za-z0-9_])(?:agb-?code|lanr|bsnr|gmc(?:\s*(?:no|number|reg))?|big(?:-?registratie)?-?(?:nummer|nr)|zorgverlener(?:snummer)?|lebenslange\s*arztnummer|betriebsst[äa]ttennummer|rpps|adeli|pwz)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
119
119
|
// Company / trade register. "Companies House" and "Company Registration
|
|
120
120
|
// Number" are written out: the tail allows a run-on *or* one qualifier word,
|
|
121
121
|
// never a second word, so a three-word label needs to be spelled.
|
|
122
122
|
[EntityType.CHAMBER_OF_COMMERCE,
|
|
123
|
-
/(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|companies\s*house(?:\s*(?:registration|reg|no|number))?|company\s*registration(?:\s*(?:number|no))?|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
123
|
+
/(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|companies\s*house(?:\s*(?:registration|reg|no|number))?|company\s*registration(?:\s*(?:number|no))?|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer|regon|krs)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
124
124
|
// VAT.
|
|
125
125
|
[EntityType.VAT,
|
|
126
126
|
/(?<![A-Za-z0-9_])(?:vat|btw|tva|ust|iva|moms|alv|mwst|dič|vsk)(?:(?:[A-Za-z0-9_.\-]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
@@ -130,6 +130,15 @@ export const CUES = [
|
|
|
130
130
|
[EntityType.BANK_ACCOUNT,
|
|
131
131
|
/(?<![A-Za-z0-9_])(?:sort\s*code|account\s*(?:no|number)|rekeningnummer|kontonummer|numéro\s*de\s*compte|sort-?code)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
132
132
|
// Passport.
|
|
133
|
+
// Residence permit. No permit patterns exist, so a permit number was filed
|
|
134
|
+
// under whichever ID, passport or phone pattern fitted it (rules-engine#53).
|
|
135
|
+
// The tail admits the Belgian card category ("nr. B 565992336"), unmasked.
|
|
136
|
+
[EntityType.RESIDENCE_PERMIT,
|
|
137
|
+
/(?<![A-Za-z0-9_])(?:aufenthalts(?:titel|erlaubnis|karte|gestattung)|niederlassungs(?:erlaubnis|bewilligung)|verblijfs(?:vergunning|kaart|document|titel)|titre\s*de\s*s[ée]jour|carte\s*de\s*s[ée]jour|permis\s*de\s*s[ée]jour|carte\s*de\s*r[ée]sident|residence\s*(?:permit|card|document)|biometric\s*residence\s*permit|permesso\s*di\s*soggiorno|carta\s*di\s*soggiorno|kart(?:a|y|ę|ą)\s*pobytu(?:\s*cukr)?|naklejk(?:a|i|ę|ą)\s*wizow(?:a|ej|ą)|uppehållstillstånd|opholdstilladelse|oppholdstillatelse|oleskelulupa)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*(?:[A-Z]\s+)?$/i],
|
|
138
|
+
// Driving licence: Polish labels; the document number has the passport
|
|
139
|
+
// shape, so without this cue it lost the span to the passport rule (rules-engine#77).
|
|
140
|
+
[EntityType.DRIVERS_LICENSE,
|
|
141
|
+
/(?<![A-Za-z0-9_])(?:(?:nr\s*dokumentu\s*)?(?:prawo|prawa|prawem)\s*jazdy|numer\s*blankietu\s*prawa\s*jazdy)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
133
142
|
[EntityType.PASSPORT,
|
|
134
143
|
/(?<![A-Za-z0-9_])(?:passport(?:\s*(?:no|number))?|paspoort(?:nummer)?|reisepass(?:nummer)?|passeport|passnummer)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
135
144
|
// BIC / SWIFT.
|
|
@@ -146,7 +155,7 @@ export const CUES = [
|
|
|
146
155
|
// only thing that identifies one, which is why the engine emits this type
|
|
147
156
|
// from here and nowhere else.
|
|
148
157
|
[EntityType.INTERNAL_ID,
|
|
149
|
-
/(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge(?:\s*(?:no|number))?|betriebsstätten(?:nr|nummer)|grundstücks(?:nr|nummer)|dossier-?(?:nr|nummer)|aktenzeichen|osobní\s*(?:číslo\s*)?zaměstnance)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
158
|
+
/(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge(?:\s*(?:no|number))?|betriebsstätten(?:nr|nummer)|grundstücks(?:nr|nummer)|dossier-?(?:nr|nummer)|aktenzeichen|schade(?:claim|nummer|dossier)|schadens?-?(?:nummer|nr)|num[ée]ro\s*de\s*sinistre|numero\s*(?:di\s*)?sinistro|n[úu]mero\s*de\s*siniestro|skade(?:nummer|sag)|claim\s*(?:no|number|id)|collar|osobní\s*(?:číslo\s*)?zaměstnance)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
150
159
|
// Credential. Typed SECRET because that is the type this SDK has; the
|
|
151
160
|
// training pipeline's report asks for CREDENTIAL, which does not exist here.
|
|
152
161
|
[EntityType.SECRET,
|
|
@@ -192,6 +201,13 @@ const RETYPABLE = new Set([
|
|
|
192
201
|
* "Betriebsstättennr. 260326822" both landed, each reported as Czech.
|
|
193
202
|
*/
|
|
194
203
|
const RETYPABLE_UNCORROBORATED = new Set([EntityType.NATIONAL_ID]);
|
|
204
|
+
/**
|
|
205
|
+
* Retypable to RESIDENCE_PERMIT by a permit label touching the span, whatever
|
|
206
|
+
* the country support: a German eAT number fits the identity-card pattern and a
|
|
207
|
+
* Dutch permit number the passport pattern (rules-engine#53). Not a Spanish
|
|
208
|
+
* NIE, which the canon keeps NATIONAL_ID.
|
|
209
|
+
*/
|
|
210
|
+
const RETYPABLE_BY_PERMIT_LABEL = new Set([EntityType.NATIONAL_ID, EntityType.PASSPORT]);
|
|
195
211
|
/**
|
|
196
212
|
* Types a label can assert on its own.
|
|
197
213
|
*
|
|
@@ -205,7 +221,7 @@ export const CUE_TARGETS = new Set([
|
|
|
205
221
|
EntityType.HEALTH_INSURANCE, EntityType.HEALTHCARE_PROVIDER,
|
|
206
222
|
EntityType.CHAMBER_OF_COMMERCE, EntityType.VAT, EntityType.POSTAL_CODE,
|
|
207
223
|
EntityType.PASSPORT, EntityType.INTERNAL_ID, EntityType.BANK_ACCOUNT,
|
|
208
|
-
EntityType.SECRET,
|
|
224
|
+
EntityType.SECRET, EntityType.RESIDENCE_PERMIT,
|
|
209
225
|
]);
|
|
210
226
|
/**
|
|
211
227
|
* Which of those a label may also *rescue* — re-admit after its checksum
|
|
@@ -242,13 +258,17 @@ export const RESCUE_TARGETS = new Set([
|
|
|
242
258
|
* `countryScore` is the document's support for the country the winning pattern
|
|
243
259
|
* came from; see RETYPABLE_UNCORROBORATED.
|
|
244
260
|
*/
|
|
245
|
-
export function retypedBy(text, start, entityType, countryScore) {
|
|
261
|
+
export function retypedBy(text, start, entityType, countryScore, country = null) {
|
|
246
262
|
const type = String(entityType);
|
|
247
|
-
|
|
248
|
-
|
|
263
|
+
const general = RETYPABLE.has(type)
|
|
264
|
+
|| (RETYPABLE_UNCORROBORATED.has(type) && countryScore === 0);
|
|
265
|
+
const byPermit = RETYPABLE_BY_PERMIT_LABEL.has(type) && country !== "ES";
|
|
266
|
+
if (!general && !byPermit)
|
|
249
267
|
return null;
|
|
250
268
|
const cued = cuedType(text, start);
|
|
251
269
|
if (cued === null || cued === entityType || !CUE_TARGETS.has(String(cued)))
|
|
252
270
|
return null;
|
|
271
|
+
if (!general && cued !== EntityType.RESIDENCE_PERMIT)
|
|
272
|
+
return null;
|
|
253
273
|
return cued;
|
|
254
274
|
}
|
package/dist/esm/rules/engine.js
CHANGED
|
@@ -242,12 +242,20 @@ export class RuleEngine {
|
|
|
242
242
|
if (this.matcher.validate(m)) {
|
|
243
243
|
validated.push({ match: m, hasValidValidator: m.patternDef.validator !== null });
|
|
244
244
|
}
|
|
245
|
-
else if (m.patternDef.validator !== null
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
if (
|
|
249
|
-
|
|
245
|
+
else if (m.patternDef.validator !== null) {
|
|
246
|
+
const cued = RESCUE_TARGETS.has(String(m.patternDef.entityType))
|
|
247
|
+
&& cuedType(text, m.start) === m.patternDef.entityType;
|
|
248
|
+
if (!m.patternDef.requiresContext) {
|
|
249
|
+
failedSpans.push({ start: m.start, end: m.end, code: m.countryCode,
|
|
250
|
+
etype: String(m.patternDef.entityType) });
|
|
251
|
+
if (cued)
|
|
252
|
+
rescued.push(m);
|
|
253
|
+
}
|
|
254
|
+
else if (cued) {
|
|
255
|
+
// A label-gated pattern is rescued too, but only by a cue touching the
|
|
256
|
+
// value; it casts no demotion zone (rules-engine#75).
|
|
250
257
|
rescued.push(m);
|
|
258
|
+
}
|
|
251
259
|
}
|
|
252
260
|
}
|
|
253
261
|
// Evidence pass: work out which countries this document belongs to, from
|
|
@@ -457,7 +465,7 @@ export class RuleEngine {
|
|
|
457
465
|
// the value is filed as a Cypriot identity number. Dropping it also clears
|
|
458
466
|
// outOfScope, which would otherwise claim the detection belongs to a
|
|
459
467
|
// country the caller did not declare while naming no country at all.
|
|
460
|
-
const retyped = retypedBy(text, match.start, entityType, countryScores.get(match.countryCode) ?? 0);
|
|
468
|
+
const retyped = retypedBy(text, match.start, entityType, countryScores.get(match.countryCode) ?? 0, match.countryCode);
|
|
461
469
|
if (retyped !== null) {
|
|
462
470
|
entityType = retyped;
|
|
463
471
|
country = null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EntityType } from "../types.js";
|
|
2
|
-
import { isRegisteredBic } from "./bicRegistry.js";
|
|
2
|
+
import { isKnownInstitution, isRegisteredBic } from "./bicRegistry.js";
|
|
3
3
|
import { DE_DISTRICT_CODES } from "./deDistricts.js";
|
|
4
4
|
const CONTEXT_CHARS = 150;
|
|
5
5
|
function getContext(text, start, end) {
|
|
@@ -21,10 +21,11 @@ const CURRENCY_WORD = "EUR|USD|GBP|CHF|ISK|SEK|NOK|DKK|PLN|CZK|HUF|RON|BGN|HRK|"
|
|
|
21
21
|
const CURRENCY_AFTER = new RegExp(`^\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
22
22
|
const CURRENCY_COMMA_AFTER = new RegExp(`^[.,]\\d{1,2}\\s*(?:(?:${CURRENCY_SYMBOL})|(?:${CURRENCY_WORD})${_UWB})`, "i");
|
|
23
23
|
const CURRENCY_BEFORE = new RegExp(`(?:(?:${CURRENCY_SYMBOL})|\\b(?:${CURRENCY_WORD}))\\s*$`, "i");
|
|
24
|
-
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)
|
|
24
|
+
const AMOUNT_LABEL_BEFORE = /\b(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð|Importe|Importo|Valore|Wartość|Kwota|Částka|Összeg|Sum|Beloop|Prix|Preis|Price|Loyer|Miete|Huur|Rent|Betrag)(?:\s+\w+)?\s*:?\s*$/i; // second word separate: not "Rent"+"enversicherungsnummer" (rules-engine#111)
|
|
25
25
|
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
26
26
|
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
27
|
-
|
|
27
|
+
// Horizontal space only: a unit is on the number's own line (rules-engine#51).
|
|
28
|
+
const UNIT_AFTER = /^[ \t]*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
28
29
|
// Same missing-boundary flaw: "ref" matched the tail of "kortref".
|
|
29
30
|
const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr|ticket|incident\s*(?:report|nr|no)?|case\s*(?:nr|no|id)?|zaaknummer|meldingsnummer|Vorgangsnummer|Vorgang|Störungsmeldung|saksnummer|ärendenummer|sagsnummer|asianumero|Factuur\s*n[ro]?\.?|Nota\s*n[ro]?\.?)\s*[:.#]?\s*$/i;
|
|
30
31
|
// "#" before a number marks a reference in every language in scope — a ticket,
|
|
@@ -32,9 +33,19 @@ const REFERENCE_BEFORE = /\b(?:dossier|ref\.?|referentie|reference|référence|f
|
|
|
32
33
|
const HASH_BEFORE = /#\s*$/;
|
|
33
34
|
// A short uppercase tag hyphenated to the number: "IR-43433", "INC-2024".
|
|
34
35
|
const REF_PREFIX_BEFORE = /(?:^|[\s([])[A-Z]{2,5}-$/;
|
|
36
|
+
// The cross-border address form, "CH-8004 Zürich": a country code hyphenated to
|
|
37
|
+
// a postal code, told apart from a reference tag by position -- it opens an
|
|
38
|
+
// address line and a place name follows (rules-engine#58).
|
|
39
|
+
const COUNTRY_PREFIX_OPENS_LINE = /(?:^|\n|,)[ \t]*([A-Z]{2})-$/;
|
|
40
|
+
const POSTAL_PREFIX_COUNTRIES = new Set([
|
|
41
|
+
"AT", "BE", "BG", "CH", "CY", "CZ", "DE", "DK", "EE", "EL", "ES", "FI",
|
|
42
|
+
"FR", "GB", "GR", "HR", "HU", "IE", "IS", "IT", "LI", "LT", "LU", "LV",
|
|
43
|
+
"MT", "NL", "NO", "PL", "PT", "RO", "SE", "SI", "SK", "UK",
|
|
44
|
+
]);
|
|
35
45
|
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
36
|
-
|
|
37
|
-
const
|
|
46
|
+
// An operator on the same line only: a bullet or "---" on the next line is not a minus sign (rules-engine#91).
|
|
47
|
+
const MATH_BEFORE = /[=+\-×÷*/][^\S\r\n]*$/;
|
|
48
|
+
const MATH_AFTER = /^[^\S\r\n]*[=+\-×÷*/]/;
|
|
38
49
|
// "A-1010", "B-2000", "L-1234", "CH-8000", "D-10115": a country prefix on a
|
|
39
50
|
// postal code, not a minus sign.
|
|
40
51
|
const COUNTRY_PREFIX_HYPHEN = /(?:^|[^A-Za-z0-9])[A-Z]{1,2}-\s*$/;
|
|
@@ -60,10 +71,11 @@ const NUMERIC_TYPES = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityT
|
|
|
60
71
|
// Month names in the languages this engine covers. Long, but it is data: the
|
|
61
72
|
// alternative is a proximity heuristic, and proximity is what caused the defect
|
|
62
73
|
// this guards against. Kept in sync with the Python SDK's suppressors.py.
|
|
63
|
-
const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
74
|
+
export const MONTH_NAMES = "january|february|march|april|may|june|july|august|september|october|november|december" +
|
|
64
75
|
"|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" +
|
|
65
76
|
"|januar|jänner|februar|märz|mai|juni|juli|oktober|dezember|dez|okt|mrz" +
|
|
66
77
|
"|janvier|février|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre" +
|
|
78
|
+
"|janv|févr|fevr|avr|juil|déc" +
|
|
67
79
|
"|januari|februari|maart|mei|augustus|mrt" +
|
|
68
80
|
"|gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|dicembre" +
|
|
69
81
|
"|enero|febrero|abril|mayo|junio|julio|septiembre|octubre|noviembre|diciembre" +
|
|
@@ -285,7 +297,19 @@ function suppressReference(text, match) {
|
|
|
285
297
|
if (REFERENCE_BEFORE.test(before))
|
|
286
298
|
return true;
|
|
287
299
|
const adjacent = text.slice(Math.max(0, match.start - 8), match.start);
|
|
288
|
-
|
|
300
|
+
if (HASH_BEFORE.test(adjacent))
|
|
301
|
+
return true;
|
|
302
|
+
if (!REF_PREFIX_BEFORE.test(adjacent))
|
|
303
|
+
return false;
|
|
304
|
+
return !(match.patternDef.entityType === EntityType.POSTAL_CODE &&
|
|
305
|
+
isCountryPrefixedAddress(text, match.start, match.end));
|
|
306
|
+
}
|
|
307
|
+
/** "CH-8004 Zürich" on an address line, not the reference "IT-20431". */
|
|
308
|
+
function isCountryPrefixedAddress(text, start, end) {
|
|
309
|
+
const found = COUNTRY_PREFIX_OPENS_LINE.exec(text.slice(Math.max(0, start - 12), start));
|
|
310
|
+
return (found !== null &&
|
|
311
|
+
POSTAL_PREFIX_COUNTRIES.has(found[1]) &&
|
|
312
|
+
CITY_AFTER.test(text.slice(end, end + 24)));
|
|
289
313
|
}
|
|
290
314
|
function suppressLegal(text, match) {
|
|
291
315
|
const applicable = LEGAL_TYPES;
|
|
@@ -338,6 +362,8 @@ function suppressRedactionPlaceholder(_text, match) {
|
|
|
338
362
|
return KNOWN_TYPE_NAMES.has(name);
|
|
339
363
|
}
|
|
340
364
|
const RECENT_YEAR = /^(?:19[5-9]\d|20[0-3]\d)$/;
|
|
365
|
+
// Any year a written date can carry: the date-construction test applies to all of them (rules-engine#111).
|
|
366
|
+
const DATE_YEAR = /^(?:1[89]\d\d|20\d\d)$/;
|
|
341
367
|
const POSTAL_CONTEXT_NEAR = /(?:postcode|postal|code\s*postal|PLZ|Postleitzahl|postnummer|postinumero|póstnúmer|zip|straat|straße|strasse|rue\s|via\s|calle\s|rua\s|ulica|utca|street|avenue|laan\s|weg\s|plein|adres|adresse|address|woonplaats|wonende|woonachtig|gevestigd|domicili|demeurant|résidant|residant|bosatt|bopæl|wohnhaft|ansässig|stad\b|ville\b|city\b|Stadt|città|ciudad|cidade|miasto|město|város)/i;
|
|
342
368
|
const DATE_KEYWORD_NEAR = /(?:født|fødselsdato|fødsel|Fødselsdato|född|födelsedatum|födelsedag|syntynyt|syntymäaika|fæddur|fæðingardagur|geboren|geboortedatum|Geburtsdatum|nascido|nacido|data di nascita|nato il|nata il|Tiltr[æa]delsesdato|Tiltredelsesdato|datum|date\b|Datum|jaar|year|année|Jahr|since|sinds|depuis|seit|\d{2}\.\d{2}\.|(?:januar|februar|marts|april|maj|juni|juli|august|september|oktober|november|december|januari|februari|mars|mei|juin|juillet|août))/i;
|
|
343
369
|
// A sentence boundary: a full stop, question or exclamation mark followed by
|
|
@@ -367,14 +393,27 @@ function sentenceAround(text, start, end, reach = 160) {
|
|
|
367
393
|
const POSTAL_LABEL_TOUCHING = new RegExp(`(?:${POSTAL_CONTEXT_NEAR.source})[\\s:.\\-]{0,4}$`, "i");
|
|
368
394
|
// A capitalised place name after the code: "2000 Antwerpen". A following label
|
|
369
395
|
// ("1970 Fødselsnummer:") is excluded by the colon.
|
|
370
|
-
|
|
396
|
+
// `\p{L}\p{N}_` rather than `\w`: JS `\w` is ASCII-only, so "8004 Zürich" failed
|
|
397
|
+
// at the ü here while Python's Unicode `\w` accepted it (rules-engine#58).
|
|
398
|
+
const CITY_AFTER = /^[ \t](?![\p{L}\p{N}_]+\s*:)[A-ZÀ-ÞŁŠŽ][\p{L}\p{N}_\-']{2,}/u;
|
|
371
399
|
// Every position is local to the value or inside its own sentence. Proximity to
|
|
372
400
|
// an address *word* anywhere in a 300-character window is what the old rescue
|
|
373
401
|
// asked, and that is a neighbourhood rather than a position (rules-engine#32).
|
|
402
|
+
/**
|
|
403
|
+
* The comma of an address line: the word before it is a place name or a number.
|
|
404
|
+
* "myocardial infarction, 2016" is a year in a list (rules-engine#111).
|
|
405
|
+
*/
|
|
406
|
+
const ADDRESS_COMMA = /(?<!\S)(?:[A-ZÀ-Þ\d][^\s,]*|[^\s,]*\]),[ \t]*$/;
|
|
407
|
+
/**
|
|
408
|
+
* A redaction placeholder: "[POSTAL_CODE]" holds the words the address test
|
|
409
|
+
* looks for, so on masked text it made every year in its sentence a postcode
|
|
410
|
+
* (rules-engine#111). A masked value is no address word.
|
|
411
|
+
*/
|
|
412
|
+
const MASKED_VALUE = /\[[A-Z][A-Z_]*\]/g;
|
|
374
413
|
function yearSitsInAddressStructure(text, start, end) {
|
|
375
414
|
const before = text.slice(Math.max(0, start - 28), start);
|
|
376
415
|
// "Amsterdam, 2026" -- the comma of an address line.
|
|
377
|
-
if (
|
|
416
|
+
if (ADDRESS_COMMA.test(before))
|
|
378
417
|
return true;
|
|
379
418
|
// "PLZ: 2011" -- a postal label with nothing but punctuation between.
|
|
380
419
|
if (POSTAL_LABEL_TOUCHING.test(before))
|
|
@@ -386,13 +425,13 @@ function yearSitsInAddressStructure(text, start, end) {
|
|
|
386
425
|
// 12, rustige ligging in 2018, vlakbij openbaar vervoer." is an Antwerp
|
|
387
426
|
// postal code in prose; the false positives put the year and the address in
|
|
388
427
|
// different sentences.
|
|
389
|
-
return POSTAL_CONTEXT_NEAR.test(sentenceAround(text, start, end));
|
|
428
|
+
return POSTAL_CONTEXT_NEAR.test(sentenceAround(text, start, end).replace(MASKED_VALUE, " "));
|
|
390
429
|
}
|
|
391
430
|
function suppressYearAsPostal(text, match) {
|
|
392
431
|
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
393
432
|
return false;
|
|
394
433
|
const clean = match.text.trim();
|
|
395
|
-
if (!
|
|
434
|
+
if (!DATE_YEAR.test(clean))
|
|
396
435
|
return false;
|
|
397
436
|
// A date construction *touching* the candidate settles it, whatever the rest
|
|
398
437
|
// of the document says. Without this, the postal-context test below rescued
|
|
@@ -404,6 +443,8 @@ function suppressYearAsPostal(text, match) {
|
|
|
404
443
|
return true;
|
|
405
444
|
if (DATE_AFTER.test(text.slice(match.end, match.end + 8)))
|
|
406
445
|
return true;
|
|
446
|
+
if (!RECENT_YEAR.test(clean))
|
|
447
|
+
return false;
|
|
407
448
|
const [before, after] = getContext(text, match.start, match.end);
|
|
408
449
|
const context = before + after;
|
|
409
450
|
// Keep as postal code only when the value sits in address *structure* --
|
|
@@ -411,10 +452,6 @@ function suppressYearAsPostal(text, match) {
|
|
|
411
452
|
// words was the defect rather than the keyword list.
|
|
412
453
|
if (yearSitsInAddressStructure(text, match.start, match.end))
|
|
413
454
|
return false;
|
|
414
|
-
// Keep if preceded by comma+space (address pattern: "Amsterdam, 2026")
|
|
415
|
-
const immediateBefore = text.slice(Math.max(0, match.start - 3), match.start);
|
|
416
|
-
if (/,\s*$/.test(immediateBefore))
|
|
417
|
-
return false;
|
|
418
455
|
// Suppress if date keyword nearby (birth date, employment date, etc.)
|
|
419
456
|
if (DATE_KEYWORD_NEAR.test(context))
|
|
420
457
|
return true;
|
|
@@ -457,12 +494,26 @@ const TOWN_AFTER = /^\s*[A-ZÀ-ÞĀ-Ž]/;
|
|
|
457
494
|
* between the marker and the number, so allow exactly one.
|
|
458
495
|
*/
|
|
459
496
|
const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|durchwahl|doorkiesnummer|tst|nebenstelle)\.?\s*(?:[a-zà-ÿ]{2,10}\s*)?[:.]?\s*$/i;
|
|
497
|
+
/**
|
|
498
|
+
* What else a bare digit run after these is (rules-engine#111): a head count
|
|
499
|
+
* ("Employees: 214"), a standard ("ISO/IEC 27001"), a product model with an
|
|
500
|
+
* inner capital ("Dell OptiPlex 7090"). Not the digits after "[PHONE]": the
|
|
501
|
+
* rest of a number an earlier pass cut short, safer masked as a postcode.
|
|
502
|
+
*/
|
|
503
|
+
const COUNT_LABEL_BEFORE = /(?<![A-Za-z0-9_])(?:employees|headcount|staff|mitarbeiter(?:innen)?|beschäftigte|werknemers|medewerkers|employés|salariés|effectifs?|dipendenti|empleados|pracowników|zaměstnanců)\s*[:=]?\s*$/i;
|
|
504
|
+
const NOT_POSTAL_BEFORE = /(?<![A-Za-z0-9_])(?:(?:ISO(?:\/IEC)?|IEC|EN|DIN(?:\s+EN)?|NEN(?:-EN)?|NF\s+EN|BS(?:\s+EN)?|PN-EN|ÖNORM)[\s\-]*|[A-Za-z]*[a-z][A-Z][A-Za-z]*\s+)$/;
|
|
505
|
+
/** A thousands group of an amount: the "750" of "£28,750,000". */
|
|
506
|
+
const GROUP_BEFORE = /\d[,.]$/;
|
|
507
|
+
const GROUP_AFTER = /^[,.]\d{3}(?!\d)/;
|
|
460
508
|
/** Suppress a postal code the word in front of it rules out. */
|
|
461
509
|
function suppressPostalAfterDisqualifyingWord(text, match) {
|
|
462
510
|
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
463
511
|
return false;
|
|
464
512
|
const before = text.slice(Math.max(0, match.start - 24), match.start);
|
|
465
|
-
if (EXTENSION_BEFORE.test(before))
|
|
513
|
+
if (EXTENSION_BEFORE.test(before) || COUNT_LABEL_BEFORE.test(before)
|
|
514
|
+
|| NOT_POSTAL_BEFORE.test(before))
|
|
515
|
+
return true;
|
|
516
|
+
if (GROUP_BEFORE.test(before) && GROUP_AFTER.test(text.slice(match.end, match.end + 5)))
|
|
466
517
|
return true;
|
|
467
518
|
return RECENT_YEAR.test(match.text.trim())
|
|
468
519
|
&& (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
|
|
@@ -473,10 +524,18 @@ function suppressPhoneServiceNumber(_text, match) {
|
|
|
473
524
|
return false;
|
|
474
525
|
return SERVICE_NUMBER.test(match.text);
|
|
475
526
|
}
|
|
476
|
-
|
|
527
|
+
// A date and the hour of a time, "07.10.2026 08" out of "07.10.2026 08:42":
|
|
528
|
+
// the French phone pattern took it as five pairs (rules-engine#101). Only when
|
|
529
|
+
// the minutes follow, so the stamp is certain.
|
|
530
|
+
const DATE_THEN_HOUR = /^\d{2}([-/.])\d{2}\1\d{4}\s+\d{1,2}$/;
|
|
531
|
+
const MINUTES_AFTER = /^:\d{2}(?!\d)/;
|
|
532
|
+
function suppressPhoneDateOverlap(text, match) {
|
|
477
533
|
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
478
534
|
return false;
|
|
479
|
-
|
|
535
|
+
const candidate = match.text.trim();
|
|
536
|
+
if (DATE_PATTERN_FULL.test(candidate))
|
|
537
|
+
return true;
|
|
538
|
+
return DATE_THEN_HOUR.test(candidate) && MINUTES_AFTER.test(text.slice(match.end, match.end + 3));
|
|
480
539
|
}
|
|
481
540
|
// Standards and classification prefixes that are plate-shaped once a letter and
|
|
482
541
|
// digits follow: "ATC-N06", "ICD-O3". None is a German district code, but the
|
|
@@ -489,6 +548,16 @@ const NOT_CITY_CODES = new Set([
|
|
|
489
548
|
"BE", "DE", "EU", "NL", "LU", "WS", "SS", "IP",
|
|
490
549
|
]);
|
|
491
550
|
const CURRENCY_PLATE = /^(?:EUR|USD|GBP|CHF|SEK|NOK|DKK|ISK|CZK|PLN|HUF|RON|BGN|HRK)\s/i;
|
|
551
|
+
const ALNUM = /[\p{L}\p{N}]/u;
|
|
552
|
+
/** Characters that join a token to more of an identifier (rules-engine#50, #81). */
|
|
553
|
+
const ID_CONNECTORS = "-/._+";
|
|
554
|
+
/** A reference marker directly before the token: "#FR-S2", "№ …", "n° …". */
|
|
555
|
+
const REFERENCE_MARK_BEFORE = /(?:#|№|(?<![A-Za-z])[nN][°º])\s?$/;
|
|
556
|
+
/** A connector glues [start, end) to a letter or digit outside it. */
|
|
557
|
+
function joinedToMore(text, start, end) {
|
|
558
|
+
return ((end + 1 < text.length && ID_CONNECTORS.includes(text[end]) && ALNUM.test(text[end + 1])) ||
|
|
559
|
+
(start >= 2 && ID_CONNECTORS.includes(text[start - 1]) && ALNUM.test(text[start - 2])));
|
|
560
|
+
}
|
|
492
561
|
function suppressPlateInCompound(text, match) {
|
|
493
562
|
if (match.patternDef.entityType !== EntityType.LICENSE_PLATE)
|
|
494
563
|
return false;
|
|
@@ -500,6 +569,18 @@ function suppressPlateInCompound(text, match) {
|
|
|
500
569
|
if (/[A-Za-zÄÖÜäöüß]{2,}-$/.test(before))
|
|
501
570
|
return true;
|
|
502
571
|
}
|
|
572
|
+
// A plate is a token of its own: joined by a connector to more of an
|
|
573
|
+
// identifier ("TF-284-KL-00874", "FR-S2/2026") it is a segment of a reference
|
|
574
|
+
// (rules-engine#50, #81). A spaced dash or slash does not join.
|
|
575
|
+
if (joinedToMore(text, match.start, match.end))
|
|
576
|
+
return true;
|
|
577
|
+
// After a reference marker ("Ref #FR-S2") it is a reference, unless a plate
|
|
578
|
+
// cue says otherwise (rules-engine#81).
|
|
579
|
+
if (REFERENCE_MARK_BEFORE.test(text.slice(Math.max(0, match.start - 4), match.start))) {
|
|
580
|
+
const [b, a] = getContext(text, match.start, match.end);
|
|
581
|
+
if (!PLATE_CUE_NEAR.test(b + a))
|
|
582
|
+
return true;
|
|
583
|
+
}
|
|
503
584
|
const matched = match.text.trim();
|
|
504
585
|
const parts = matched.split(/[\s\-]+/);
|
|
505
586
|
// A standards or classification reference, not a plate — unless a plate cue
|
|
@@ -829,10 +910,34 @@ function suppressBicWithoutEvidence(text, match, scratch) {
|
|
|
829
910
|
// The token itself is blanked out so it cannot vouch for itself.
|
|
830
911
|
const unit = structuralUnit(text, match.start, match.end).split(token).join(" ".repeat(token.length));
|
|
831
912
|
if (BIC_KEYWORD.test(unit) || IBAN_SHAPE.test(unit) || BANK_BLOCK.test(unit)) {
|
|
913
|
+
if (bicShapedWord(text, match.start, match.end, token))
|
|
914
|
+
return true;
|
|
832
915
|
return occursAsLowercaseWord(text, token, scratch);
|
|
833
916
|
}
|
|
834
917
|
return true;
|
|
835
918
|
}
|
|
919
|
+
/** A personal title right before the token: "Dr. HOFFMANN", "Mme JANSSENS". */
|
|
920
|
+
const TITLE_BEFORE = /(?<![A-Za-z])(?:dr|prof|mr|mrs|ms|mme|mlle|m|herr|frau|dhr|mevr|sig|sra?)\.?\s+$/i;
|
|
921
|
+
const LETTER = /\p{L}/u;
|
|
922
|
+
/** A BIC/SWIFT label touching the token: "BIC: ", "SWIFT-Code:\n", "Code SWIFT : ", "BIC Bank A: ". */
|
|
923
|
+
const BIC_LABEL_BEFORE = /(?<![A-Za-z])(?:bic|swift)(?:[\s/\-]*(?:code|swift|bic))?(?:\s+bank\s+\w{1,3})?[\s:.\-()]*$/i;
|
|
924
|
+
/**
|
|
925
|
+
* A registry miss sitting where only a word or a name can (rules-engine#57):
|
|
926
|
+
* joined to a word by a hyphen ("NGUYEN-HOFFMANN"), right after a personal
|
|
927
|
+
* title ("Dr. HOFFMANN"), or letters only (eight, or eleven without an `XXX`
|
|
928
|
+
* branch: "BETALING", "MAANDELIJKS") that the GLEIF mapping does not know and
|
|
929
|
+
* no BIC/SWIFT label touches.
|
|
930
|
+
*/
|
|
931
|
+
function bicShapedWord(text, start, end, token) {
|
|
932
|
+
if ((start >= 2 && text[start - 1] === "-" && LETTER.test(text[start - 2])) ||
|
|
933
|
+
(text[end] === "-" && end + 1 < text.length && LETTER.test(text[end + 1])))
|
|
934
|
+
return true;
|
|
935
|
+
if (TITLE_BEFORE.test(text.slice(Math.max(0, start - 8), start)))
|
|
936
|
+
return true;
|
|
937
|
+
return /^[A-Za-z]+$/.test(token) && !(token.length === 11 && token.endsWith("XXX"))
|
|
938
|
+
&& !isKnownInstitution(token)
|
|
939
|
+
&& !BIC_LABEL_BEFORE.test(text.slice(Math.max(0, start - 30), start));
|
|
940
|
+
}
|
|
836
941
|
// ── Postal code: digits belonging to a longer identifier ────────────────
|
|
837
942
|
// An identifier label immediately before the digits. A postal code is never
|
|
838
943
|
// introduced this way; an SVNr, policy number or service number always is.
|
|
@@ -852,7 +957,14 @@ const DIALLING_PREFIX_BEFORE = /\+\d{1,3}[\d\s\-().]*$/;
|
|
|
852
957
|
// read as a country-prefixed address: in "PV-2026-LU-09143" the `LU` is
|
|
853
958
|
// preceded by `-` and the German five-digit postal pattern kept `09143`
|
|
854
959
|
// (rules-engine#31).
|
|
855
|
-
const COUNTRY_PREFIXED = /(?:^|[^A-Za-z0-9\-])[A-Z]{1,
|
|
960
|
+
const COUNTRY_PREFIXED = /(?:^|[^A-Za-z0-9\-])([A-Z]{1,3})$/;
|
|
961
|
+
// Postal country prefixes and the length of that country's postcode: any one or two capitals
|
|
962
|
+
// used to pass, so "K-40817" and "B-51823" were country-prefixed postcodes (rules-engine#111).
|
|
963
|
+
const POSTAL_PREFIX_DIGITS = { A: 4, AT: 4, B: 4, BE: 4, BG: 4, CH: 4, CY: 4, CZ: 5, D: 5, DE: 5, DK: 4, E: 5, EE: 5, ES: 5, F: 5, FI: 5, FIN: 5, FR: 5, GR: 5, H: 4, HR: 5, I: 5, IT: 5, L: 4, LT: 5, LU: 4, LV: 4, N: 4, NL: 4, NO: 4, P: 4, PL: 5, PT: 4, RO: 6, S: 5, SE: 5, SI: 4, SK: 5, SLO: 4 };
|
|
964
|
+
function countryPrefixed(before, digits) {
|
|
965
|
+
const m = COUNTRY_PREFIXED.exec(before);
|
|
966
|
+
return m !== null && POSTAL_PREFIX_DIGITS[m[1]] === digits.length;
|
|
967
|
+
}
|
|
856
968
|
/**
|
|
857
969
|
* Suppress bare digit runs that belong to a longer number, not an address.
|
|
858
970
|
*
|
|
@@ -868,8 +980,15 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
868
980
|
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
869
981
|
return false;
|
|
870
982
|
const clean = match.text.trim();
|
|
871
|
-
if (!/^\d+$/.test(clean))
|
|
983
|
+
if (!/^\d+$/.test(clean)) {
|
|
984
|
+
// A structured form glued on by a hyphen is the tail of a reference: "CPS-HR-2026-019" (#111).
|
|
985
|
+
const glued = text.slice(Math.max(0, match.start - 2), match.start);
|
|
986
|
+
if (glued.length === 2 && glued[1] === "-" && /[A-Za-z0-9]/.test(glued[0])) {
|
|
987
|
+
const m = COUNTRY_PREFIXED.exec(text.slice(0, match.start - 1));
|
|
988
|
+
return !(m && m[1] in POSTAL_PREFIX_DIGITS);
|
|
989
|
+
}
|
|
872
990
|
return false;
|
|
991
|
+
}
|
|
873
992
|
const beforeChar = match.start > 0 ? text[match.start - 1] : "";
|
|
874
993
|
const afterChar = match.end < text.length ? text[match.end] : "";
|
|
875
994
|
const prevPrev = match.start >= 2 ? text[match.start - 2] : "";
|
|
@@ -888,7 +1007,10 @@ function suppressPostalInLongerIdentifier(text, match) {
|
|
|
888
1007
|
return true;
|
|
889
1008
|
if (afterChar === "-" && /\d/.test(nextNext))
|
|
890
1009
|
return true;
|
|
891
|
-
|
|
1010
|
+
// A hyphenated chain goes on after it: "réf. : L-2025-HD-0412" is a reference (#111).
|
|
1011
|
+
if (/^-[A-Za-z0-9]+-/.test(text.slice(match.end, match.end + 12)))
|
|
1012
|
+
return true;
|
|
1013
|
+
if (beforeChar === "-" && !countryPrefixed(text.slice(0, match.start - 1), clean))
|
|
892
1014
|
return true;
|
|
893
1015
|
// A further digit group on the same line: "1268 040390", "1234 5678 925",
|
|
894
1016
|
// "+43 664 8213 907". Horizontal whitespace only — a digit on the *next*
|
|
@@ -952,10 +1074,156 @@ function suppressRequiresContext(text, match) {
|
|
|
952
1074
|
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
953
1075
|
fillsADelimitedField(text, match.start, match.end))
|
|
954
1076
|
return false;
|
|
1077
|
+
if (DATE_LABEL_TYPES.has(match.patternDef.entityType)) {
|
|
1078
|
+
return !labelsThisDate(text, match.start, match.end, match.patternDef.contextKeywords, match.patternDef.entityType);
|
|
1079
|
+
}
|
|
955
1080
|
const [before, after] = getContext(text, match.start, match.end);
|
|
956
1081
|
const context = (before + " " + after).toLowerCase();
|
|
957
1082
|
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
958
1083
|
}
|
|
1084
|
+
/** Date types decided by a label: DOB and DATE_OF_DEATH share one date shape. */
|
|
1085
|
+
const DATE_LABEL_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
|
|
1086
|
+
/** Any common numeric date, to tell whether a label belongs to another date. */
|
|
1087
|
+
// A masked date counts as one: the gateway's rules pass reads SDK-masked text (rules-engine#111).
|
|
1088
|
+
// ... and a written-month date, now a DOB shape of its own (rules-engine#115).
|
|
1089
|
+
const ANY_DATE = new RegExp(String.raw `\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}|\[(?:dob|date_of_death)\]|(?<![A-Za-z0-9_])\d{1,2}\.?\s+(?:` + MONTH_NAMES + String.raw `)\.?\s+\d{4}(?![A-Za-z0-9_])`, "i");
|
|
1090
|
+
/**
|
|
1091
|
+
* True when one of `keywords` is this date's own label: before the date with no
|
|
1092
|
+
* other date between, or after it inside the date's own sentence. Anywhere in
|
|
1093
|
+
* the window typed every date from whichever date label the document carried
|
|
1094
|
+
* (rules-engine#52).
|
|
1095
|
+
*/
|
|
1096
|
+
/** A date followed by a list separator: an item of the list a label introduces. */
|
|
1097
|
+
const LIST_ITEM = /\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}\s*(?:,|;|–|&|\band\b|\ben\b|\bet\b|\bund\b|\bor\b|\bof\b|\bou\b|\boder\b|\brespectievelijk\b|\bbzw\.?)\s*/g;
|
|
1098
|
+
/** Punctuation and at most two short words ("e.g.", "bv.", "op") between label and date. */
|
|
1099
|
+
const LABEL_TAIL = /^[\s:.,()\-]*(?:\p{L}{1,4}(?:\.\p{L}{1,2})*\.?(?:[\s:.,()\-]+|$)){0,2}$/u;
|
|
1100
|
+
/** The birth sign "°" directly before the date, but not the number sign "n°". */
|
|
1101
|
+
const BIRTH_SIGN = /(?<![a-z])°\s?$/;
|
|
1102
|
+
/** No other date between a label and the date, except items of its list. */
|
|
1103
|
+
function nothingButAListBetween(gap) {
|
|
1104
|
+
if (!ANY_DATE.test(gap))
|
|
1105
|
+
return true;
|
|
1106
|
+
const rest = gap.replace(LIST_ITEM, " ");
|
|
1107
|
+
return !ANY_DATE.test(rest) && LABEL_TAIL.test(rest);
|
|
1108
|
+
}
|
|
1109
|
+
/**
|
|
1110
|
+
* Where `keyword` occurs in the lower-cased `haystack`, or -1. An all-capitals
|
|
1111
|
+
* keyword is an abbreviation and counts only as a word of its own: "DOB" is a
|
|
1112
|
+
* label, the "dob" in Polish "dobrowolnej" is not (rules-engine#100); a plural
|
|
1113
|
+
* "s" may follow ("DOBs e.g. …"). Other
|
|
1114
|
+
* keywords still match inside a longer word, which inflected forms need.
|
|
1115
|
+
*/
|
|
1116
|
+
function keywordAt(haystack, keyword, last) {
|
|
1117
|
+
const kw = keyword.toLowerCase();
|
|
1118
|
+
if (keyword !== keyword.toUpperCase() || keyword === kw) {
|
|
1119
|
+
return last ? haystack.lastIndexOf(kw) : haystack.indexOf(kw);
|
|
1120
|
+
}
|
|
1121
|
+
const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
|
|
1122
|
+
const hits = [];
|
|
1123
|
+
for (let i = haystack.indexOf(kw); i !== -1; i = haystack.indexOf(kw, i + 1)) {
|
|
1124
|
+
let next = i + kw.length;
|
|
1125
|
+
if (haystack[i - 1] === "[" && haystack[next] === "]")
|
|
1126
|
+
continue; // a placeholder ("[DOB]"), not a label (#111)
|
|
1127
|
+
if (haystack[next] === "s")
|
|
1128
|
+
next++;
|
|
1129
|
+
if (!isLetter(haystack.slice(i - 1, i)) && !isLetter(haystack.slice(next, next + 1))) {
|
|
1130
|
+
hits.push(i);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
if (hits.length === 0)
|
|
1134
|
+
return -1;
|
|
1135
|
+
return last ? hits[hits.length - 1] : hits[0];
|
|
1136
|
+
}
|
|
1137
|
+
function labelsThisDate(text, start, end, keywords, entityType) {
|
|
1138
|
+
const [before, after] = getContext(text, start, end);
|
|
1139
|
+
const lowerBefore = before.toLowerCase();
|
|
1140
|
+
const lowerAfter = after.toLowerCase();
|
|
1141
|
+
// A label on an earlier line counts only when the date opens its own line or
|
|
1142
|
+
// the label was asked as a question (a call transcript's answer turn);
|
|
1143
|
+
// a column header is a fallback and never overrules a label of the date's own.
|
|
1144
|
+
const lineStart = lowerBefore.lastIndexOf("\n") + 1;
|
|
1145
|
+
const opensLine = lowerBefore.slice(lineStart).replace(/[ \t:|;*\-•(\[]/g, "") === "";
|
|
1146
|
+
if (entityType === EntityType.DOB && BIRTH_SIGN.test(lowerBefore))
|
|
1147
|
+
return true;
|
|
1148
|
+
for (const keyword of keywords) {
|
|
1149
|
+
const kw = keyword.toLowerCase();
|
|
1150
|
+
const i = keywordAt(lowerBefore, keyword, true);
|
|
1151
|
+
if (i >= 0 && nothingButAListBetween(lowerBefore.slice(i + kw.length))
|
|
1152
|
+
&& (i >= lineStart || opensLine
|
|
1153
|
+
|| lowerBefore.slice(i + kw.length, lineStart).includes("?")))
|
|
1154
|
+
return true;
|
|
1155
|
+
const j = keywordAt(lowerAfter, keyword, false);
|
|
1156
|
+
if (j >= 0) {
|
|
1157
|
+
const gap = lowerAfter.slice(0, j);
|
|
1158
|
+
if (!ANY_DATE.test(gap) && !SENTENCE_BREAK.test(gap)
|
|
1159
|
+
&& !LEADS_TO_A_DATE.test(lowerAfter.slice(j + kw.length)))
|
|
1160
|
+
return true;
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
const header = columnHeader(text, start);
|
|
1164
|
+
return header !== null && keywords.some(kw => keywordAt(header, kw, false) >= 0);
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* A label running straight into a date of its own introduces that date, not the
|
|
1168
|
+
* one before it ("Verstorben am 01.02.2020, geboren am 12.03.1940").
|
|
1169
|
+
*/
|
|
1170
|
+
// ... or into a value already masked ("né le [DOB]"): the label is that value's (rules-engine#111).
|
|
1171
|
+
const LEADS_TO_A_DATE = new RegExp(String.raw `^[\s:.\-]*(?:[a-zà-ÿ]{1,3}[\s:.]+)?(?:\d{1,4}[/.\-]\d{1,2}[/.\-]\d{2,4}|\[[a-z_]+\]|\d{1,2}\.?\s+(?:` + MONTH_NAMES + String.raw `)\.?\s+\d{4})`, "i");
|
|
1172
|
+
/** Field separators a table row may use. */
|
|
1173
|
+
const COLUMN_SEPARATORS = [";", "|", "\t", ","];
|
|
1174
|
+
/**
|
|
1175
|
+
* The lower-cased header cell above the value, if it sits in a table: in a table
|
|
1176
|
+
* the column decides the label, not proximity. A header is one of the three
|
|
1177
|
+
* lines above, split by the row's separator into the same number of cells.
|
|
1178
|
+
*/
|
|
1179
|
+
function columnHeader(text, start) {
|
|
1180
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
1181
|
+
const lineEndAt = text.indexOf("\n", start);
|
|
1182
|
+
const line = text.slice(lineStart, lineEndAt >= 0 ? lineEndAt : text.length);
|
|
1183
|
+
// The three lines above, by lastIndexOf: splitting the whole prefix was
|
|
1184
|
+
// quadratic in the document, once per date candidate.
|
|
1185
|
+
let aboveStart = lineStart - 1;
|
|
1186
|
+
for (let k = 0; k < 3 && aboveStart >= 0; k++) {
|
|
1187
|
+
aboveStart = aboveStart === 0 ? -1 : text.lastIndexOf("\n", aboveStart - 1);
|
|
1188
|
+
}
|
|
1189
|
+
const above = lineStart ? text.slice(aboveStart + 1, Math.max(0, lineStart - 1)).split("\n") : [];
|
|
1190
|
+
for (const sep of COLUMN_SEPARATORS) {
|
|
1191
|
+
if (!line.includes(sep))
|
|
1192
|
+
continue;
|
|
1193
|
+
const cells = line.split(sep);
|
|
1194
|
+
const column = text.slice(lineStart, start).split(sep).length - 1;
|
|
1195
|
+
for (const header of [...above].reverse()) {
|
|
1196
|
+
// A "label : value" field with a comma in its value is not a header (#111).
|
|
1197
|
+
if (header.includes(":"))
|
|
1198
|
+
continue;
|
|
1199
|
+
const heads = header.split(sep);
|
|
1200
|
+
if (heads.length === cells.length && !ANY_DATE.test(header))
|
|
1201
|
+
return heads[column].toLowerCase();
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
return null;
|
|
1205
|
+
}
|
|
1206
|
+
/** A dotted MAC shape glued to more of a reference is part of it: the tail of
|
|
1207
|
+
* "WSO-II.6151.4471.2025" (rules-engine#105). Dotted form only. */
|
|
1208
|
+
function suppressMacInReference(text, match) {
|
|
1209
|
+
if (match.patternDef.entityType !== EntityType.MAC_ADDRESS || !match.text.includes("."))
|
|
1210
|
+
return false;
|
|
1211
|
+
return joinedToMore(text, match.start, match.end);
|
|
1212
|
+
}
|
|
1213
|
+
/** A card number is never carved out of a longer run of digit groups: a
|
|
1214
|
+
* four-digit group joined by the card's own separator on either side means the
|
|
1215
|
+
* run is longer, e.g. a Polish account number (rules-engine#106). */
|
|
1216
|
+
function suppressCardInsideDigitRun(text, match) {
|
|
1217
|
+
if (match.patternDef.entityType !== EntityType.CREDIT_CARD)
|
|
1218
|
+
return false;
|
|
1219
|
+
const sep = [...match.text].find(c => !/\d/.test(c)) ?? "";
|
|
1220
|
+
if (!sep)
|
|
1221
|
+
return false;
|
|
1222
|
+
const s = sep.replace(/[-\\^$.*+?()[\]{}|]/g, "\\$&");
|
|
1223
|
+
const before = text.slice(Math.max(0, match.start - 6), match.start);
|
|
1224
|
+
const after = text.slice(match.end, match.end + 6);
|
|
1225
|
+
return new RegExp(`(?<!\\d)\\d{4}${s}$`).test(before) || new RegExp(`^${s}\\d{4}(?!\\d)`).test(after);
|
|
1226
|
+
}
|
|
959
1227
|
const TYPE_SUPPRESSORS = {
|
|
960
1228
|
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneServiceNumber, suppressPhoneInsideAccountRun, suppressPhoneDateOverlap, suppressPhoneAsNumberRange],
|
|
961
1229
|
[EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
@@ -967,6 +1235,8 @@ const TYPE_SUPPRESSORS = {
|
|
|
967
1235
|
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
|
968
1236
|
[EntityType.SECRET]: [suppressSecretOverStructured, suppressSecretNotASecret],
|
|
969
1237
|
[EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
1238
|
+
[EntityType.MAC_ADDRESS]: [suppressMacInReference],
|
|
1239
|
+
[EntityType.CREDIT_CARD]: [suppressCardInsideDigitRun],
|
|
970
1240
|
};
|
|
971
1241
|
export function shouldSuppress(text, match, scratch) {
|
|
972
1242
|
if (suppressSequential(text, match))
|