euredact 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -4
- package/dist/cjs/rules/cues.js +101 -29
- package/dist/cjs/rules/engine.js +51 -4
- package/dist/cjs/rules/suppressors.js +48 -1
- package/dist/esm/rules/cues.d.ts +42 -12
- package/dist/esm/rules/cues.js +100 -28
- package/dist/esm/rules/engine.d.ts +32 -0
- package/dist/esm/rules/engine.js +51 -5
- package/dist/esm/rules/suppressors.js +48 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -230,10 +230,37 @@ SOCIAL_HANDLE SECRET INTERNAL_ID OTHER
|
|
|
230
230
|
|
|
231
231
|
`INTERNAL_ID` — an employee, badge or customer number tied to a person — is
|
|
232
232
|
emitted **only** when an explicit label names it (`medarbejdernummer:`,
|
|
233
|
-
`Personalnummer:`, `Employee No
|
|
234
|
-
is no shape for one: without the label, a digit
|
|
235
|
-
any other. The type exists so that a labelled
|
|
236
|
-
correctly instead of being claimed by the phone
|
|
233
|
+
`Personalnummer:`, `Employee No:`, `Betriebsstättennr.`, `Badge`). There is no
|
|
234
|
+
pattern for one, because there is no shape for one: without the label, a digit
|
|
235
|
+
run is not distinguishable from any other. The type exists so that a labelled
|
|
236
|
+
employee number is filed correctly instead of being claimed by the phone
|
|
237
|
+
pattern.
|
|
238
|
+
|
|
239
|
+
### What a label in front of a value decides
|
|
240
|
+
|
|
241
|
+
A label touching the left edge of a value decides what that value is called.
|
|
242
|
+
The label may be the abbreviation or the word it abbreviates — `BSN:` and
|
|
243
|
+
`Burgerservicenummer:` both reach `NATIONAL_ID` — and it may be the official
|
|
244
|
+
name in the document's own language:
|
|
245
|
+
|
|
246
|
+
| label | type |
|
|
247
|
+
|---|---|
|
|
248
|
+
| `Companies House Registration:`, `Company Registration Number:` | `CHAMBER_OF_COMMERCE` |
|
|
249
|
+
| `Sozialversicherungsnummer:` | `SSN` |
|
|
250
|
+
| `Passport No.:`, `Paspoortnummer:` | `PASSPORT` |
|
|
251
|
+
| `AGB-code:`, `LANR`, `GMC Number:` | `HEALTHCARE_PROVIDER` |
|
|
252
|
+
| `Medical Card No.:` | `HEALTH_INSURANCE` |
|
|
253
|
+
| `sort code`, `account number` | `BANK_ACCOUNT` |
|
|
254
|
+
| `TAN-activatiecode`, `activation code` | `SECRET` |
|
|
255
|
+
|
|
256
|
+
A label can also *rule a type out*. A four-digit run is not a postal code when
|
|
257
|
+
a founding or payment participle introduces it (`Opgericht in 2016`,
|
|
258
|
+
`Fondée en 2017`) or when it sits in a telephone parenthetical (`(toest. 3841)`,
|
|
259
|
+
`(ext. 2219)`). Postal codes that merely look like years are unaffected —
|
|
260
|
+
Antwerp's `2018` in `rustige ligging in 2018` is still a `POSTAL_CODE`.
|
|
261
|
+
|
|
262
|
+
A label never moves a span; it only decides the label. Which characters are
|
|
263
|
+
masked is unchanged either way.
|
|
237
264
|
|
|
238
265
|
For custom patterns, `entityType` is a plain string (e.g. `"EMPLOYEE_ID"`).
|
|
239
266
|
|
package/dist/cjs/rules/cues.js
CHANGED
|
@@ -34,18 +34,34 @@
|
|
|
34
34
|
* the *tail* of an ordinary word. "NIS" matched "tālru**nis**", the Latvian
|
|
35
35
|
* for telephone, and suppressed 653 real phone numbers; "iva" matched
|
|
36
36
|
* "Pr**iva**t:" and read a personal e-mail address as a VAT number.
|
|
37
|
-
* *
|
|
38
|
-
*
|
|
37
|
+
* * Neither the qualifier nor the run-on may use `\w`. The qualifier is a
|
|
38
|
+
* negated ASCII class, fixed for this in 0.3.8; `[^\W\d_]` would reject
|
|
39
|
+
* "εταιρειας" here for the same ASCII reason. The run-on was *not* fixed then
|
|
40
|
+
* and went on diverging for another release: Slovak "telefón 0956550012"
|
|
41
|
+
* needs it to absorb "efón" after `tel`, and Bulgarian
|
|
42
|
+
* "пощенски кодове 4000-4999" needs "ове" after `пощенски код`. Python's
|
|
43
|
+
* Unicode `\w` did; this SDK's ASCII `\w` did not, so one engine said `PHONE`
|
|
44
|
+
* and the other `NATIONAL_ID` on the same document. They were the last two
|
|
45
|
+
* type divergences on the corpus, and only the type-aware half of
|
|
46
|
+
* `make parity` could see them.
|
|
39
47
|
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
48
|
+
* The run-on is spelled `(?:[A-Za-z0-9_]|[^\x00-\x7F])` — an ASCII word
|
|
49
|
+
* character, or any non-ASCII character — rather than as a negated class of
|
|
50
|
+
* separators. A negated class also admits `'`, `%`, `&`, `@` and the rest of
|
|
51
|
+
* ASCII punctuation, which `\w` never did, and widening it that far was
|
|
52
|
+
* measured at 46 extra false positives with hints and 57 blind over the
|
|
53
|
+
* 152,300-document corpus. This is `\w` plus the non-ASCII letters `\w`
|
|
54
|
+
* should have matched all along, and nothing else.
|
|
55
|
+
*
|
|
56
|
+
* Every label ends in `(?:<run-on>|\s+<one word>)`: the label either runs on
|
|
57
|
+
* into a longer word ("telefoon" → "telefoonnummer") **or** takes one qualifier
|
|
58
|
+
* word naming whose identifier it is ("ΑΦΜ εταιρειας:", "Tel Nr:") — never both.
|
|
43
59
|
* Allowing both let "nie" grow into the surname "Nieminen" and then swallow the
|
|
44
60
|
* real label in "Matti Nieminen\nDOB: 08.05.1994", reading a date of birth as a
|
|
45
61
|
* Spanish NIE: 52 false positives across the corpus.
|
|
46
62
|
*/
|
|
47
63
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
-
exports.CUE_TARGETS = exports.CUES = exports.CUE_WINDOW = void 0;
|
|
64
|
+
exports.RESCUE_TARGETS = exports.CUE_TARGETS = exports.CUES = exports.CUE_WINDOW = void 0;
|
|
49
65
|
exports.cuedType = cuedType;
|
|
50
66
|
exports.retypedBy = retypedBy;
|
|
51
67
|
const types_js_1 = require("../types.js");
|
|
@@ -54,7 +70,7 @@ const types_js_1 = require("../types.js");
|
|
|
54
70
|
* the value it introduces, which is the lesson of the postal-code year defect,
|
|
55
71
|
* where a cue 150 characters away licensed every number in the window.
|
|
56
72
|
*/
|
|
57
|
-
exports.CUE_WINDOW =
|
|
73
|
+
exports.CUE_WINDOW = 32;
|
|
58
74
|
/**
|
|
59
75
|
* (entity type, label pattern). Ordered: the first match wins, so the narrower
|
|
60
76
|
* label comes first where two could read the same text.
|
|
@@ -64,46 +80,63 @@ exports.CUES = [
|
|
|
64
80
|
// front of a span that no phone pattern matched says the document is laid out
|
|
65
81
|
// unusually, not that the value is a phone number.
|
|
66
82
|
[types_js_1.EntityType.PHONE,
|
|
67
|
-
/(?<![A-Za-z0-9_])(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell|tālrunis|τηλέφωνο|τηλ)(
|
|
83
|
+
/(?<![A-Za-z0-9_])(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell|tālrunis|τηλέφωνο|τηλ)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
68
84
|
// National identity number.
|
|
69
85
|
[types_js_1.EntityType.NATIONAL_ID,
|
|
70
|
-
/(?<![A-Za-z0-9_])(?:bsn|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|dni|rr|nn|nir|insz|niss|nis|matricule|ausweisnummer|personalausweis|osobní\s*číslo|ЕГН|EGN)(
|
|
86
|
+
/(?<![A-Za-z0-9_])(?:bsn|burgerservicenummer(?:\s*\(bsn\))?|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|dni|rr|nn|nir|insz|niss|nis|matricule|ausweisnummer|personalausweis|osobní\s*číslo|ЕГН|EGN)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
71
87
|
// Cyprus: "ID number" in both official languages, plus the Greek abbreviation
|
|
72
88
|
// ΑΔΤ (Αστυνομική Ταυτότητα). Neither has a phone reading.
|
|
73
89
|
[types_js_1.EntityType.NATIONAL_ID,
|
|
74
|
-
/(?<![A-Za-z0-9_])(?:αρ\.?\s*ταυτότητας|ταυτότητας|αδτ|kimlik\s*numaras[ıi])(
|
|
90
|
+
/(?<![A-Za-z0-9_])(?:αρ\.?\s*ταυτότητας|ταυτότητας|αδτ|kimlik\s*numaras[ıi])(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
75
91
|
// Social security.
|
|
76
92
|
[types_js_1.EntityType.SSN,
|
|
77
|
-
/(?<![A-Za-z0-9_])(?:ssn|social\s*security|sozialversicherungsnummer|svnr|sofi)(
|
|
93
|
+
/(?<![A-Za-z0-9_])(?:ssn|social\s*security|sozialversicherungsnummer|svnr|sofi)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
78
94
|
// Tax number. Greece's ΑΦΜ is issued by the tax authority; the identity-card
|
|
79
|
-
// number is the ΑΔΤ, above.
|
|
80
|
-
// cued ΑΦΜ, which is wrong independently of the phone problem.
|
|
95
|
+
// number is the ΑΔΤ, above.
|
|
81
96
|
[types_js_1.EntityType.TAX_ID,
|
|
82
|
-
/(?<![A-Za-z0-9_])(?:αφμ|α\.φ\.μ\.|steuer-?id|steuernummer|st\.?-?nr|stnr|tin|finanzamt\s+ist|tax\s*(?:id|no|number)|daňové\s*číslo|numer\s*podatkowy)(
|
|
83
|
-
// Health insurance.
|
|
97
|
+
/(?<![A-Za-z0-9_])(?:αφμ|α\.φ\.μ\.|steuer-?id|steuernummer|st\.?-?nr|stnr|tin|finanzamt\s+ist|tax\s*(?:id|no|number)|daňové\s*číslo|numer\s*podatkowy)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
98
|
+
// Health insurance: the insured person.
|
|
84
99
|
[types_js_1.EntityType.HEALTH_INSURANCE,
|
|
85
|
-
/(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer)(
|
|
86
|
-
//
|
|
100
|
+
/(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer|medical\s*card(?:\s*(?:no|number))?)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
101
|
+
// Healthcare provider: the clinician or practice, not the patient — the Dutch
|
|
102
|
+
// AGB-code, the German LANR, the UK GMC number.
|
|
103
|
+
[types_js_1.EntityType.HEALTHCARE_PROVIDER,
|
|
104
|
+
/(?<![A-Za-z0-9_])(?:agb-?code|lanr|bsnr|gmc(?:\s*(?:no|number|reg))?|big-?nummer|zorgverlener(?:snummer)?)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
105
|
+
// Company / trade register. "Companies House" and "Company Registration
|
|
106
|
+
// Number" are written out: the tail allows a run-on *or* one qualifier word,
|
|
107
|
+
// never a second word, so a three-word label needs to be spelled.
|
|
87
108
|
[types_js_1.EntityType.CHAMBER_OF_COMMERCE,
|
|
88
|
-
/(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer)(
|
|
109
|
+
/(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|companies\s*house(?:\s*(?:registration|reg|no|number))?|company\s*registration(?:\s*(?:number|no))?|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
89
110
|
// VAT.
|
|
90
111
|
[types_js_1.EntityType.VAT,
|
|
91
|
-
/(?<![A-Za-z0-9_])(?:vat|btw|tva|ust|iva|moms|alv|mwst|dič|vsk)(?:[
|
|
112
|
+
/(?<![A-Za-z0-9_])(?:vat|btw|tva|ust|iva|moms|alv|mwst|dič|vsk)(?:(?:[A-Za-z0-9_.\-]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
113
|
+
// Bank account. A UK sort code is NN-NN-NN, also a licence-plate shape, so
|
|
114
|
+
// "sort code 20-45-91" was typed LICENSE_PLATE with the label in front of it.
|
|
115
|
+
// Retype target only — a failed mod-97 is never rescued by a label.
|
|
116
|
+
[types_js_1.EntityType.BANK_ACCOUNT,
|
|
117
|
+
/(?<![A-Za-z0-9_])(?:sort\s*code|account\s*(?:no|number)|rekeningnummer|kontonummer|numéro\s*de\s*compte|sort-?code)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
118
|
+
// Passport.
|
|
119
|
+
[types_js_1.EntityType.PASSPORT,
|
|
120
|
+
/(?<![A-Za-z0-9_])(?:passport(?:\s*(?:no|number))?|paspoort(?:nummer)?|reisepass(?:nummer)?|passeport|passnummer)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
92
121
|
// BIC / SWIFT.
|
|
93
122
|
[types_js_1.EntityType.BIC,
|
|
94
|
-
/(?<![A-Za-z0-9_])(?:bic|swift)[
|
|
123
|
+
/(?<![A-Za-z0-9_])(?:bic|swift)(?:[A-Za-z0-9_\/]|[^\x00-\x7F])*\s*[:.\-]?\s*\(?\s*$/i],
|
|
95
124
|
// Postal code. A postal code loses every contested span on priority (it is
|
|
96
125
|
// the weakest evidence the engine has), so without a cue entry
|
|
97
126
|
// "Cod postal: 040171" was handed to a German phone pattern.
|
|
98
127
|
[types_js_1.EntityType.POSTAL_CODE,
|
|
99
|
-
/(?<![A-Za-z0-9_])(?:cod\s*po[sș]tal|postnummer|postnr|postleitzahl|plz
|
|
128
|
+
/(?<![A-Za-z0-9_])(?:cod\s*po[sș]tal|postnummer|postnr|postleitzahl|plz(?:-?bereiche?)?|postcode|code\s*postal|codice\s*postale|kod\s*pocztowy|ps[čc]|τ\.?κ\.?|пощенски\s*код)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
100
129
|
// Internal / employee identifier. The canon calls these INTERNAL_ID and
|
|
101
130
|
// annotates them with an LLM, so the engine has no pattern for them — but it
|
|
102
131
|
// does have a phone pattern that was happily claiming them. The cue is the
|
|
103
132
|
// only thing that identifies one, which is why the engine emits this type
|
|
104
133
|
// from here and nowhere else.
|
|
105
134
|
[types_js_1.EntityType.INTERNAL_ID,
|
|
106
|
-
/(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge
|
|
135
|
+
/(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge(?:\s*(?:no|number))?|betriebsstätten(?:nr|nummer)|grundstücks(?:nr|nummer)|dossier-?(?:nr|nummer)|aktenzeichen|osobní\s*(?:číslo\s*)?zaměstnance)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
136
|
+
// Credential. Typed SECRET because that is the type this SDK has; the
|
|
137
|
+
// training pipeline's report asks for CREDENTIAL, which does not exist here.
|
|
138
|
+
[types_js_1.EntityType.SECRET,
|
|
139
|
+
/(?<![A-Za-z0-9_])(?:tan-?activatiecode|activatiecode|activation\s*code|verificatiecode|verification\s*code|otp|pincode)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
107
140
|
];
|
|
108
141
|
/**
|
|
109
142
|
* The entity type a label immediately before `start` names, or null.
|
|
@@ -129,22 +162,56 @@ const RETYPABLE = new Set([
|
|
|
129
162
|
types_js_1.EntityType.PHONE, types_js_1.EntityType.POSTAL_CODE, types_js_1.EntityType.LICENSE_PLATE,
|
|
130
163
|
]);
|
|
131
164
|
/**
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
165
|
+
* Retypable, but only when the document gives its country no support at all.
|
|
166
|
+
*
|
|
167
|
+
* These carry a checksum, so normally the pattern outranks any label. But a
|
|
168
|
+
* checksum only says the digits fit *some* national scheme, and a weak one fits
|
|
169
|
+
* by luck — the same reasoning that demotes an uncorroborated validated
|
|
170
|
+
* candidate to priority 1 in `engine.ts`. At a country score of 0.0 the engine
|
|
171
|
+
* is already saying "this passed a Czech checksum in a document with nothing
|
|
172
|
+
* Czech about it", and an explicit "Passport No.:" is the better evidence.
|
|
173
|
+
*
|
|
174
|
+
* Restricted to zero country support on purpose: a Dutch BSN in a Dutch
|
|
175
|
+
* document scores above zero and stays a NATIONAL_ID whatever label sits near
|
|
176
|
+
* it, so this cannot relabel a domestic identifier — only the coincidental
|
|
177
|
+
* foreign ones, which is where "Passport No.: 512847603" and
|
|
178
|
+
* "Betriebsstättennr. 260326822" both landed, each reported as Czech.
|
|
179
|
+
*/
|
|
180
|
+
const RETYPABLE_UNCORROBORATED = new Set([types_js_1.EntityType.NATIONAL_ID]);
|
|
181
|
+
/**
|
|
182
|
+
* Types a label can assert on its own.
|
|
135
183
|
*
|
|
136
184
|
* PHONE is absent on purpose: "Tel:" in front of something no phone pattern
|
|
137
185
|
* matched means the document is laid out oddly, not that the value is a phone
|
|
138
|
-
* number. So are EMAIL,
|
|
186
|
+
* number. So are EMAIL, DOB, VIN and the rest — those carry their own
|
|
139
187
|
* structure, and a nearby word is not entitled to overrule it.
|
|
188
|
+
*/
|
|
189
|
+
exports.CUE_TARGETS = new Set([
|
|
190
|
+
types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID,
|
|
191
|
+
types_js_1.EntityType.HEALTH_INSURANCE, types_js_1.EntityType.HEALTHCARE_PROVIDER,
|
|
192
|
+
types_js_1.EntityType.CHAMBER_OF_COMMERCE, types_js_1.EntityType.VAT, types_js_1.EntityType.POSTAL_CODE,
|
|
193
|
+
types_js_1.EntityType.PASSPORT, types_js_1.EntityType.INTERNAL_ID, types_js_1.EntityType.BANK_ACCOUNT,
|
|
194
|
+
types_js_1.EntityType.SECRET,
|
|
195
|
+
]);
|
|
196
|
+
/**
|
|
197
|
+
* Which of those a label may also *rescue* — re-admit after its checksum
|
|
198
|
+
* failed. Until 0.3.9 this was the same set as CUE_TARGETS and it cannot stay
|
|
199
|
+
* that way: re-typing a span the engine already decided to mask is cheap to be
|
|
200
|
+
* wrong about, while rescuing one asserts that a value which failed its check
|
|
201
|
+
* digit is a real identifier anyway.
|
|
140
202
|
*
|
|
141
203
|
* BIC's absence is what keeps the rescue honest. Its validator is a *registry
|
|
142
204
|
* lookup*, not a checksum, so a failure means "no such bank", which no label can
|
|
143
205
|
* talk you out of. Rescuing it read 34 unlisted codes behind a "BIC:" label as
|
|
144
206
|
* real ones. The same reasoning keeps CREDIT_CARD and BANK_ACCOUNT out: Luhn and
|
|
145
207
|
* mod-97 are strong enough that a failure really does mean "not one of these".
|
|
208
|
+
* BANK_ACCOUNT joined CUE_TARGETS in 0.3.9 so "sort code 20-45-91" stops being a
|
|
209
|
+
* LICENSE_PLATE, and is kept out here for exactly the 0.3.8 reason.
|
|
210
|
+
*
|
|
211
|
+
* HEALTHCARE_PROVIDER and SECRET are absent because neither is checksummed;
|
|
212
|
+
* there is no failed validation for a label to overrule in the first place.
|
|
146
213
|
*/
|
|
147
|
-
exports.
|
|
214
|
+
exports.RESCUE_TARGETS = new Set([
|
|
148
215
|
types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID,
|
|
149
216
|
types_js_1.EntityType.HEALTH_INSURANCE, types_js_1.EntityType.CHAMBER_OF_COMMERCE,
|
|
150
217
|
types_js_1.EntityType.VAT, types_js_1.EntityType.POSTAL_CODE, types_js_1.EntityType.PASSPORT,
|
|
@@ -157,9 +224,14 @@ exports.CUE_TARGETS = new Set([
|
|
|
157
224
|
* without them fired 52 times and got roughly 30 of those wrong — EMAIL read as
|
|
158
225
|
* VAT, a date of birth as a phone number, a VIN as VAT — against 20 real fixes.
|
|
159
226
|
* With them: 14 re-typings, every one of them PHONE to a structured identifier.
|
|
227
|
+
*
|
|
228
|
+
* `countryScore` is the document's support for the country the winning pattern
|
|
229
|
+
* came from; see RETYPABLE_UNCORROBORATED.
|
|
160
230
|
*/
|
|
161
|
-
function retypedBy(text, start, entityType) {
|
|
162
|
-
|
|
231
|
+
function retypedBy(text, start, entityType, countryScore) {
|
|
232
|
+
const type = String(entityType);
|
|
233
|
+
if (!RETYPABLE.has(type)
|
|
234
|
+
&& !(RETYPABLE_UNCORROBORATED.has(type) && countryScore === 0))
|
|
163
235
|
return null;
|
|
164
236
|
const cued = cuedType(text, start);
|
|
165
237
|
if (cued === null || cued === entityType || !exports.CUE_TARGETS.has(String(cued)))
|
package/dist/cjs/rules/engine.js
CHANGED
|
@@ -35,6 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.RuleEngine = exports.COUNTRY_CODE_ALIASES = void 0;
|
|
37
37
|
exports.resolveCountryCode = resolveCountryCode;
|
|
38
|
+
exports.tieKey = tieKey;
|
|
38
39
|
const types_js_1 = require("../types.js");
|
|
39
40
|
const matchers_js_1 = require("./matchers.js");
|
|
40
41
|
const suppressors_js_1 = require("./suppressors.js");
|
|
@@ -81,6 +82,48 @@ function resolveCountryCode(countryCode) {
|
|
|
81
82
|
code = exports.COUNTRY_CODE_ALIASES[code] ?? code;
|
|
82
83
|
return code in index_js_1.COUNTRY_CONFIGS && code !== "SHARED" ? code : null;
|
|
83
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Last-resort ordering for candidates every other field has tied on.
|
|
87
|
+
*
|
|
88
|
+
* Reaching this means the engine has no evidence left: same span, no cue, same
|
|
89
|
+
* priority, the same (usually zero) country score, and the same declared scope.
|
|
90
|
+
* Something still has to win, and before this the winner was whichever candidate
|
|
91
|
+
* happened to be generated first — which is country *registration* order, and
|
|
92
|
+
* the two SDKs do not share one. Python registers alphabetically
|
|
93
|
+
* (`AT, BE, BG, CH, CY, CZ, ...`); this SDK uses a curated order
|
|
94
|
+
* (`NL, BE, DE, AT, CH, FR, ..., PT, LU, PL, ..., CZ`). So a bare digit run that
|
|
95
|
+
* several national schemes accept was filed as a Czech `NATIONAL_ID` by Python
|
|
96
|
+
* and a Portuguese `PHONE` here — identical characters masked, contradictory
|
|
97
|
+
* labels, and `make parity` compared only characters, so it stayed green through
|
|
98
|
+
* the whole of 0.3.8.
|
|
99
|
+
*
|
|
100
|
+
* The order reproduces what Python already did rather than inventing a new one,
|
|
101
|
+
* because every accuracy figure this engine has published was measured with
|
|
102
|
+
* Python resolving these ties by its registration order. Ordering on the entity
|
|
103
|
+
* type instead was tried first and re-tuned the corpus twice over: it turned the
|
|
104
|
+
* German `Steuerß12345678911` from `TAX_ID`/DE into `NATIONAL_ID`/EL, and moved
|
|
105
|
+
* 820 further spans from `DOB` to `DATE_OF_DEATH`. So the key is deliberately
|
|
106
|
+
* not total — candidates from the same country are left to the stable sort,
|
|
107
|
+
* which preserves the order that country's pattern list was written in, and both
|
|
108
|
+
* SDKs write theirs in the same order.
|
|
109
|
+
*
|
|
110
|
+
* Taken from the **pattern**, never the finished detection: this SDK re-types a
|
|
111
|
+
* cued candidate before the sort and Python re-types after it, so keying on the
|
|
112
|
+
* final type would reintroduce the divergence in a subtler form.
|
|
113
|
+
*
|
|
114
|
+
* Kept character-for-character in step with `tie_key` in `rules/engine.py`.
|
|
115
|
+
*/
|
|
116
|
+
function tieKey(countryCode) {
|
|
117
|
+
// "SHARED" and "CUSTOM" are not countries and sort as themselves under a
|
|
118
|
+
// plain alphabetical key — landing between PT and SK, rather than ahead of
|
|
119
|
+
// every country the way the registry loads them. The bucket restores that.
|
|
120
|
+
const bucket = countryCode === "SHARED" ? "0" : countryCode === "CUSTOM" ? "1" : "2";
|
|
121
|
+
// NUL separator, written as an escape so this file stays plain ASCII, and
|
|
122
|
+
// matching Python byte for byte. It must sort below every character a code
|
|
123
|
+
// or type name can contain, or a code that is a prefix of another would
|
|
124
|
+
// order differently in the two SDKs.
|
|
125
|
+
return `${bucket}\u0000${countryCode}`;
|
|
126
|
+
}
|
|
84
127
|
const warnedCountries = new Set();
|
|
85
128
|
function warnUnknownCountry(code) {
|
|
86
129
|
if (warnedCountries.has(code))
|
|
@@ -240,7 +283,7 @@ class RuleEngine {
|
|
|
240
283
|
else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
|
|
241
284
|
failedSpans.push({ start: m.start, end: m.end, code: m.countryCode,
|
|
242
285
|
etype: String(m.patternDef.entityType) });
|
|
243
|
-
if (cues_js_1.
|
|
286
|
+
if (cues_js_1.RESCUE_TARGETS.has(String(m.patternDef.entityType))
|
|
244
287
|
&& (0, cues_js_1.cuedType)(text, m.start) === m.patternDef.entityType)
|
|
245
288
|
rescued.push(m);
|
|
246
289
|
}
|
|
@@ -452,7 +495,7 @@ class RuleEngine {
|
|
|
452
495
|
// the value is filed as a Cypriot identity number. Dropping it also clears
|
|
453
496
|
// outOfScope, which would otherwise claim the detection belongs to a
|
|
454
497
|
// country the caller did not declare while naming no country at all.
|
|
455
|
-
const retyped = (0, cues_js_1.retypedBy)(text, match.start, entityType);
|
|
498
|
+
const retyped = (0, cues_js_1.retypedBy)(text, match.start, entityType, countryScores.get(match.countryCode) ?? 0);
|
|
456
499
|
if (retyped !== null) {
|
|
457
500
|
entityType = retyped;
|
|
458
501
|
country = null;
|
|
@@ -466,6 +509,7 @@ class RuleEngine {
|
|
|
466
509
|
length: match.end - match.start,
|
|
467
510
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
468
511
|
inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
|
|
512
|
+
tie: tieKey(match.countryCode),
|
|
469
513
|
tier: 0,
|
|
470
514
|
det: {
|
|
471
515
|
entityType,
|
|
@@ -504,6 +548,7 @@ class RuleEngine {
|
|
|
504
548
|
length: match.end - match.start,
|
|
505
549
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
506
550
|
inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
|
|
551
|
+
tie: tieKey(match.countryCode),
|
|
507
552
|
tier: 0,
|
|
508
553
|
det: {
|
|
509
554
|
entityType: match.patternDef.entityType,
|
|
@@ -519,7 +564,8 @@ class RuleEngine {
|
|
|
519
564
|
}
|
|
520
565
|
for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
|
|
521
566
|
candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
|
|
522
|
-
countryScore: 0, inScope: 1, tier: 0, det: d
|
|
567
|
+
countryScore: 0, inScope: 1, tier: 0, det: d,
|
|
568
|
+
tie: tieKey(d.country ?? "") });
|
|
523
569
|
}
|
|
524
570
|
const detections = this.deduplicate(candidates);
|
|
525
571
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
@@ -587,7 +633,8 @@ class RuleEngine {
|
|
|
587
633
|
b.cue - a.cue || // local cue } country-aware
|
|
588
634
|
b.priority - a.priority || // country priority } — decides the
|
|
589
635
|
b.countryScore - a.countryScore || // country evidence } LABEL only
|
|
590
|
-
b.inScope - a.inScope
|
|
636
|
+
b.inScope - a.inScope || // declared scope }
|
|
637
|
+
(a.tie < b.tie ? -1 : a.tie > b.tie ? 1 : 0)); // total } — see tieKey
|
|
591
638
|
const result = [];
|
|
592
639
|
const occupied = new Set();
|
|
593
640
|
for (const { det } of sorted) {
|
|
@@ -347,6 +347,53 @@ function suppressYearAsPostal(text, match) {
|
|
|
347
347
|
// Suppress: recent years without postal context are almost never postal codes
|
|
348
348
|
return true;
|
|
349
349
|
}
|
|
350
|
+
// ── Postal code: disqualified by the word in front of it ───────────────
|
|
351
|
+
//
|
|
352
|
+
// The mirror image of `rules/cues.ts`. There a label promotes a type; here a
|
|
353
|
+
// word refuses one. Both were reported together: a bare four-digit run becomes
|
|
354
|
+
// a POSTAL_CODE as soon as *any* real postal code establishes the country,
|
|
355
|
+
// which is to say in almost every real document, because POSTAL_CONTEXT_NEAR
|
|
356
|
+
// above then rescues it from suppressYearAsPostal. Measured on the training
|
|
357
|
+
// corpus: 55 bare years in prose ("Opgericht in 2016", "Fondée en 2017") plus
|
|
358
|
+
// telephone extensions ("(toest. 3841)", "(poste interne 3318)", "(ext. 2219)").
|
|
359
|
+
/**
|
|
360
|
+
* Prepositions that can only be temporal. Nothing is ever located "since 2018",
|
|
361
|
+
* so these disqualify a postal code on their own.
|
|
362
|
+
*/
|
|
363
|
+
const YEAR_WORD_BEFORE = /(?<![A-Za-z0-9_])(?:sinds|since|depuis|seit|siden|sedan|desde|vuonna|anno|dal)\s+$/i;
|
|
364
|
+
/**
|
|
365
|
+
* "in", "en" and "im" are *both* temporal and locative, and in exactly the
|
|
366
|
+
* countries whose postal codes are year-shaped. Belgian 2000 is Antwerp and
|
|
367
|
+
* 2018 is one of its districts, so "Rustige ligging in 2018, vlakbij openbaar
|
|
368
|
+
* vervoer" is an address, not a date — an earlier version of this rule
|
|
369
|
+
* suppressed it and lost a real postal code, which is the worse error for a
|
|
370
|
+
* redaction tool. So an ambiguous preposition needs a founding or payment
|
|
371
|
+
* participle in front of it, which is what every case in the report had:
|
|
372
|
+
* "Opgericht in 2016", "Fondée en 2017", "versé en 2025".
|
|
373
|
+
*/
|
|
374
|
+
const YEAR_VERB_BEFORE = /(?<![A-Za-z0-9_])(?:opgericht|gesticht|opgestart|founded|established|created|fond[ée]e?|cr[ée][ée]e?|gegründet|gegruendet|errichtet|grundlagt|grundad|perustettu|fundada|fundado|fondata|costituita|vers[ée]|betaald|uitbetaald|ausgezahlt|paid|geboren|born)\s+(?:in|en|im|op|the)\s+$/i;
|
|
375
|
+
/**
|
|
376
|
+
* A capitalised word immediately after keeps the candidate: "in 2000 Antwerpen"
|
|
377
|
+
* really is a postal code, and so is the "AZ" of a Dutch "1105 AZ". Without
|
|
378
|
+
* this the rule would suppress the very addresses it is meant to leave alone.
|
|
379
|
+
*/
|
|
380
|
+
const TOWN_AFTER = /^\s*[A-ZÀ-ÞĀ-Ž]/;
|
|
381
|
+
/**
|
|
382
|
+
* A telephone extension marker. "poste interne 3318" puts one qualifier word
|
|
383
|
+
* between the marker and the number, so allow exactly one.
|
|
384
|
+
*/
|
|
385
|
+
const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|durchwahl|doorkiesnummer|tst|nebenstelle)\.?\s*(?:[a-zà-ÿ]{2,10}\s*)?[:.]?\s*$/i;
|
|
386
|
+
/** Suppress a postal code the word in front of it rules out. */
|
|
387
|
+
function suppressPostalAfterDisqualifyingWord(text, match) {
|
|
388
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.POSTAL_CODE)
|
|
389
|
+
return false;
|
|
390
|
+
const before = text.slice(Math.max(0, match.start - 24), match.start);
|
|
391
|
+
if (EXTENSION_BEFORE.test(before))
|
|
392
|
+
return true;
|
|
393
|
+
return RECENT_YEAR.test(match.text.trim())
|
|
394
|
+
&& (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
|
|
395
|
+
&& !TOWN_AFTER.test(text.slice(match.end, match.end + 3));
|
|
396
|
+
}
|
|
350
397
|
function suppressPhoneServiceNumber(_text, match) {
|
|
351
398
|
if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
|
|
352
399
|
return false;
|
|
@@ -796,7 +843,7 @@ const TYPE_SUPPRESSORS = {
|
|
|
796
843
|
[types_js_1.EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
797
844
|
[types_js_1.EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
798
845
|
[types_js_1.EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
799
|
-
[types_js_1.EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier, suppressReference],
|
|
846
|
+
[types_js_1.EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalAfterDisqualifyingWord, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier, suppressReference],
|
|
800
847
|
[types_js_1.EntityType.BIC]: [suppressBicWithoutEvidence],
|
|
801
848
|
[types_js_1.EntityType.BANK_ACCOUNT]: [suppressReference],
|
|
802
849
|
[types_js_1.EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
package/dist/esm/rules/cues.d.ts
CHANGED
|
@@ -33,12 +33,28 @@
|
|
|
33
33
|
* the *tail* of an ordinary word. "NIS" matched "tālru**nis**", the Latvian
|
|
34
34
|
* for telephone, and suppressed 653 real phone numbers; "iva" matched
|
|
35
35
|
* "Pr**iva**t:" and read a personal e-mail address as a VAT number.
|
|
36
|
-
* *
|
|
37
|
-
*
|
|
36
|
+
* * Neither the qualifier nor the run-on may use `\w`. The qualifier is a
|
|
37
|
+
* negated ASCII class, fixed for this in 0.3.8; `[^\W\d_]` would reject
|
|
38
|
+
* "εταιρειας" here for the same ASCII reason. The run-on was *not* fixed then
|
|
39
|
+
* and went on diverging for another release: Slovak "telefón 0956550012"
|
|
40
|
+
* needs it to absorb "efón" after `tel`, and Bulgarian
|
|
41
|
+
* "пощенски кодове 4000-4999" needs "ове" after `пощенски код`. Python's
|
|
42
|
+
* Unicode `\w` did; this SDK's ASCII `\w` did not, so one engine said `PHONE`
|
|
43
|
+
* and the other `NATIONAL_ID` on the same document. They were the last two
|
|
44
|
+
* type divergences on the corpus, and only the type-aware half of
|
|
45
|
+
* `make parity` could see them.
|
|
38
46
|
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
47
|
+
* The run-on is spelled `(?:[A-Za-z0-9_]|[^\x00-\x7F])` — an ASCII word
|
|
48
|
+
* character, or any non-ASCII character — rather than as a negated class of
|
|
49
|
+
* separators. A negated class also admits `'`, `%`, `&`, `@` and the rest of
|
|
50
|
+
* ASCII punctuation, which `\w` never did, and widening it that far was
|
|
51
|
+
* measured at 46 extra false positives with hints and 57 blind over the
|
|
52
|
+
* 152,300-document corpus. This is `\w` plus the non-ASCII letters `\w`
|
|
53
|
+
* should have matched all along, and nothing else.
|
|
54
|
+
*
|
|
55
|
+
* Every label ends in `(?:<run-on>|\s+<one word>)`: the label either runs on
|
|
56
|
+
* into a longer word ("telefoon" → "telefoonnummer") **or** takes one qualifier
|
|
57
|
+
* word naming whose identifier it is ("ΑΦΜ εταιρειας:", "Tel Nr:") — never both.
|
|
42
58
|
* Allowing both let "nie" grow into the surname "Nieminen" and then swallow the
|
|
43
59
|
* real label in "Matti Nieminen\nDOB: 08.05.1994", reading a date of birth as a
|
|
44
60
|
* Spanish NIE: 52 false positives across the corpus.
|
|
@@ -49,7 +65,7 @@ import { EntityType } from "../types.js";
|
|
|
49
65
|
* the value it introduces, which is the lesson of the postal-code year defect,
|
|
50
66
|
* where a cue 150 characters away licensed every number in the window.
|
|
51
67
|
*/
|
|
52
|
-
export declare const CUE_WINDOW =
|
|
68
|
+
export declare const CUE_WINDOW = 32;
|
|
53
69
|
/**
|
|
54
70
|
* (entity type, label pattern). Ordered: the first match wins, so the narrower
|
|
55
71
|
* label comes first where two could read the same text.
|
|
@@ -64,22 +80,33 @@ export declare const CUES: Array<[EntityType, RegExp]>;
|
|
|
64
80
|
*/
|
|
65
81
|
export declare function cuedType(text: string, start: number): EntityType | null;
|
|
66
82
|
/**
|
|
67
|
-
* Types a label can assert on its own.
|
|
68
|
-
* in `engine.ts`, for the same reason read the other way — a label may vouch
|
|
69
|
-
* for one of these, so it may also vouch for a malformed one.
|
|
83
|
+
* Types a label can assert on its own.
|
|
70
84
|
*
|
|
71
85
|
* PHONE is absent on purpose: "Tel:" in front of something no phone pattern
|
|
72
86
|
* matched means the document is laid out oddly, not that the value is a phone
|
|
73
|
-
* number. So are EMAIL,
|
|
87
|
+
* number. So are EMAIL, DOB, VIN and the rest — those carry their own
|
|
74
88
|
* structure, and a nearby word is not entitled to overrule it.
|
|
89
|
+
*/
|
|
90
|
+
export declare const CUE_TARGETS: Set<string>;
|
|
91
|
+
/**
|
|
92
|
+
* Which of those a label may also *rescue* — re-admit after its checksum
|
|
93
|
+
* failed. Until 0.3.9 this was the same set as CUE_TARGETS and it cannot stay
|
|
94
|
+
* that way: re-typing a span the engine already decided to mask is cheap to be
|
|
95
|
+
* wrong about, while rescuing one asserts that a value which failed its check
|
|
96
|
+
* digit is a real identifier anyway.
|
|
75
97
|
*
|
|
76
98
|
* BIC's absence is what keeps the rescue honest. Its validator is a *registry
|
|
77
99
|
* lookup*, not a checksum, so a failure means "no such bank", which no label can
|
|
78
100
|
* talk you out of. Rescuing it read 34 unlisted codes behind a "BIC:" label as
|
|
79
101
|
* real ones. The same reasoning keeps CREDIT_CARD and BANK_ACCOUNT out: Luhn and
|
|
80
102
|
* mod-97 are strong enough that a failure really does mean "not one of these".
|
|
103
|
+
* BANK_ACCOUNT joined CUE_TARGETS in 0.3.9 so "sort code 20-45-91" stops being a
|
|
104
|
+
* LICENSE_PLATE, and is kept out here for exactly the 0.3.8 reason.
|
|
105
|
+
*
|
|
106
|
+
* HEALTHCARE_PROVIDER and SECRET are absent because neither is checksummed;
|
|
107
|
+
* there is no failed validation for a label to overrule in the first place.
|
|
81
108
|
*/
|
|
82
|
-
export declare const
|
|
109
|
+
export declare const RESCUE_TARGETS: Set<string>;
|
|
83
110
|
/**
|
|
84
111
|
* The type a cue overrules `entityType` with, or null to leave it alone.
|
|
85
112
|
*
|
|
@@ -87,5 +114,8 @@ export declare const CUE_TARGETS: Set<string>;
|
|
|
87
114
|
* without them fired 52 times and got roughly 30 of those wrong — EMAIL read as
|
|
88
115
|
* VAT, a date of birth as a phone number, a VIN as VAT — against 20 real fixes.
|
|
89
116
|
* With them: 14 re-typings, every one of them PHONE to a structured identifier.
|
|
117
|
+
*
|
|
118
|
+
* `countryScore` is the document's support for the country the winning pattern
|
|
119
|
+
* came from; see RETYPABLE_UNCORROBORATED.
|
|
90
120
|
*/
|
|
91
|
-
export declare function retypedBy(text: string, start: number, entityType: EntityType | string): EntityType | null;
|
|
121
|
+
export declare function retypedBy(text: string, start: number, entityType: EntityType | string, countryScore: number): EntityType | null;
|
package/dist/esm/rules/cues.js
CHANGED
|
@@ -33,12 +33,28 @@
|
|
|
33
33
|
* the *tail* of an ordinary word. "NIS" matched "tālru**nis**", the Latvian
|
|
34
34
|
* for telephone, and suppressed 653 real phone numbers; "iva" matched
|
|
35
35
|
* "Pr**iva**t:" and read a personal e-mail address as a VAT number.
|
|
36
|
-
* *
|
|
37
|
-
*
|
|
36
|
+
* * Neither the qualifier nor the run-on may use `\w`. The qualifier is a
|
|
37
|
+
* negated ASCII class, fixed for this in 0.3.8; `[^\W\d_]` would reject
|
|
38
|
+
* "εταιρειας" here for the same ASCII reason. The run-on was *not* fixed then
|
|
39
|
+
* and went on diverging for another release: Slovak "telefón 0956550012"
|
|
40
|
+
* needs it to absorb "efón" after `tel`, and Bulgarian
|
|
41
|
+
* "пощенски кодове 4000-4999" needs "ове" after `пощенски код`. Python's
|
|
42
|
+
* Unicode `\w` did; this SDK's ASCII `\w` did not, so one engine said `PHONE`
|
|
43
|
+
* and the other `NATIONAL_ID` on the same document. They were the last two
|
|
44
|
+
* type divergences on the corpus, and only the type-aware half of
|
|
45
|
+
* `make parity` could see them.
|
|
38
46
|
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
47
|
+
* The run-on is spelled `(?:[A-Za-z0-9_]|[^\x00-\x7F])` — an ASCII word
|
|
48
|
+
* character, or any non-ASCII character — rather than as a negated class of
|
|
49
|
+
* separators. A negated class also admits `'`, `%`, `&`, `@` and the rest of
|
|
50
|
+
* ASCII punctuation, which `\w` never did, and widening it that far was
|
|
51
|
+
* measured at 46 extra false positives with hints and 57 blind over the
|
|
52
|
+
* 152,300-document corpus. This is `\w` plus the non-ASCII letters `\w`
|
|
53
|
+
* should have matched all along, and nothing else.
|
|
54
|
+
*
|
|
55
|
+
* Every label ends in `(?:<run-on>|\s+<one word>)`: the label either runs on
|
|
56
|
+
* into a longer word ("telefoon" → "telefoonnummer") **or** takes one qualifier
|
|
57
|
+
* word naming whose identifier it is ("ΑΦΜ εταιρειας:", "Tel Nr:") — never both.
|
|
42
58
|
* Allowing both let "nie" grow into the surname "Nieminen" and then swallow the
|
|
43
59
|
* real label in "Matti Nieminen\nDOB: 08.05.1994", reading a date of birth as a
|
|
44
60
|
* Spanish NIE: 52 false positives across the corpus.
|
|
@@ -49,7 +65,7 @@ import { EntityType } from "../types.js";
|
|
|
49
65
|
* the value it introduces, which is the lesson of the postal-code year defect,
|
|
50
66
|
* where a cue 150 characters away licensed every number in the window.
|
|
51
67
|
*/
|
|
52
|
-
export const CUE_WINDOW =
|
|
68
|
+
export const CUE_WINDOW = 32;
|
|
53
69
|
/**
|
|
54
70
|
* (entity type, label pattern). Ordered: the first match wins, so the narrower
|
|
55
71
|
* label comes first where two could read the same text.
|
|
@@ -59,46 +75,63 @@ export const CUES = [
|
|
|
59
75
|
// front of a span that no phone pattern matched says the document is laid out
|
|
60
76
|
// unusually, not that the value is a phone number.
|
|
61
77
|
[EntityType.PHONE,
|
|
62
|
-
/(?<![A-Za-z0-9_])(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell|tālrunis|τηλέφωνο|τηλ)(
|
|
78
|
+
/(?<![A-Za-z0-9_])(?:phone|tel|telephone|téléphone|telefoon|telefon|telefono|teléfono|tlf|mobil|mobile|gsm|handy|sími|puh|móvil|cell|tālrunis|τηλέφωνο|τηλ)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
63
79
|
// National identity number.
|
|
64
80
|
[EntityType.NATIONAL_ID,
|
|
65
|
-
/(?<![A-Za-z0-9_])(?:bsn|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|dni|rr|nn|nir|insz|niss|nis|matricule|ausweisnummer|personalausweis|osobní\s*číslo|ЕГН|EGN)(
|
|
81
|
+
/(?<![A-Za-z0-9_])(?:bsn|burgerservicenummer(?:\s*\(bsn\))?|personnummer|rijksregisternummer|nationaal\s*nummer|numéro\s*national|national\s*(?:id|number)|identiteitsnummer|henkilötunnus|kennitala|cpr(?:-?nummer)?|nif|nie|dni|rr|nn|nir|insz|niss|nis|matricule|ausweisnummer|personalausweis|osobní\s*číslo|ЕГН|EGN)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
66
82
|
// Cyprus: "ID number" in both official languages, plus the Greek abbreviation
|
|
67
83
|
// ΑΔΤ (Αστυνομική Ταυτότητα). Neither has a phone reading.
|
|
68
84
|
[EntityType.NATIONAL_ID,
|
|
69
|
-
/(?<![A-Za-z0-9_])(?:αρ\.?\s*ταυτότητας|ταυτότητας|αδτ|kimlik\s*numaras[ıi])(
|
|
85
|
+
/(?<![A-Za-z0-9_])(?:αρ\.?\s*ταυτότητας|ταυτότητας|αδτ|kimlik\s*numaras[ıi])(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
70
86
|
// Social security.
|
|
71
87
|
[EntityType.SSN,
|
|
72
|
-
/(?<![A-Za-z0-9_])(?:ssn|social\s*security|sozialversicherungsnummer|svnr|sofi)(
|
|
88
|
+
/(?<![A-Za-z0-9_])(?:ssn|social\s*security|sozialversicherungsnummer|svnr|sofi)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
73
89
|
// Tax number. Greece's ΑΦΜ is issued by the tax authority; the identity-card
|
|
74
|
-
// number is the ΑΔΤ, above.
|
|
75
|
-
// cued ΑΦΜ, which is wrong independently of the phone problem.
|
|
90
|
+
// number is the ΑΔΤ, above.
|
|
76
91
|
[EntityType.TAX_ID,
|
|
77
|
-
/(?<![A-Za-z0-9_])(?:αφμ|α\.φ\.μ\.|steuer-?id|steuernummer|st\.?-?nr|stnr|tin|finanzamt\s+ist|tax\s*(?:id|no|number)|daňové\s*číslo|numer\s*podatkowy)(
|
|
78
|
-
// Health insurance.
|
|
92
|
+
/(?<![A-Za-z0-9_])(?:αφμ|α\.φ\.μ\.|steuer-?id|steuernummer|st\.?-?nr|stnr|tin|finanzamt\s+ist|tax\s*(?:id|no|number)|daňové\s*číslo|numer\s*podatkowy)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
93
|
+
// Health insurance: the insured person.
|
|
79
94
|
[EntityType.HEALTH_INSURANCE,
|
|
80
|
-
/(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer)(
|
|
81
|
-
//
|
|
95
|
+
/(?<![A-Za-z0-9_])(?:versichertennummer|krankenversicherung|kvnr|kv-?nr|mutualité|ziekenfonds|aansluitingsnummer|medical\s*card(?:\s*(?:no|number))?)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
96
|
+
// Healthcare provider: the clinician or practice, not the patient — the Dutch
|
|
97
|
+
// AGB-code, the German LANR, the UK GMC number.
|
|
98
|
+
[EntityType.HEALTHCARE_PROVIDER,
|
|
99
|
+
/(?<![A-Za-z0-9_])(?:agb-?code|lanr|bsnr|gmc(?:\s*(?:no|number|reg))?|big-?nummer|zorgverlener(?:snummer)?)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
100
|
+
// Company / trade register. "Companies House" and "Company Registration
|
|
101
|
+
// Number" are written out: the tail allows a run-on *or* one qualifier word,
|
|
102
|
+
// never a second word, so a three-word label needs to be spelled.
|
|
82
103
|
[EntityType.CHAMBER_OF_COMMERCE,
|
|
83
|
-
/(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer)(
|
|
104
|
+
/(?<![A-Za-z0-9_])(?:siren|siret|kbo|bce|kvk|ondernemingsnummer|ondernemingen\s+onder\s+nummer|kruispuntbank|numéro\s*d'entreprise|enterprise\s*number|handelsregister|companies\s*house(?:\s*(?:registration|reg|no|number))?|company\s*registration(?:\s*(?:number|no))?|company\s*(?:no|number|reg)|organisationsnummer|orgnr|i[čc]o|identifikační\s*číslo|ЕИК|eik|bulstat|cvr(?:-?nummer)?|virksomhedsnummer)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
84
105
|
// VAT.
|
|
85
106
|
[EntityType.VAT,
|
|
86
|
-
/(?<![A-Za-z0-9_])(?:vat|btw|tva|ust|iva|moms|alv|mwst|dič|vsk)(?:[
|
|
107
|
+
/(?<![A-Za-z0-9_])(?:vat|btw|tva|ust|iva|moms|alv|mwst|dič|vsk)(?:(?:[A-Za-z0-9_.\-]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
108
|
+
// Bank account. A UK sort code is NN-NN-NN, also a licence-plate shape, so
|
|
109
|
+
// "sort code 20-45-91" was typed LICENSE_PLATE with the label in front of it.
|
|
110
|
+
// Retype target only — a failed mod-97 is never rescued by a label.
|
|
111
|
+
[EntityType.BANK_ACCOUNT,
|
|
112
|
+
/(?<![A-Za-z0-9_])(?:sort\s*code|account\s*(?:no|number)|rekeningnummer|kontonummer|numéro\s*de\s*compte|sort-?code)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
113
|
+
// Passport.
|
|
114
|
+
[EntityType.PASSPORT,
|
|
115
|
+
/(?<![A-Za-z0-9_])(?:passport(?:\s*(?:no|number))?|paspoort(?:nummer)?|reisepass(?:nummer)?|passeport|passnummer)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
87
116
|
// BIC / SWIFT.
|
|
88
117
|
[EntityType.BIC,
|
|
89
|
-
/(?<![A-Za-z0-9_])(?:bic|swift)[
|
|
118
|
+
/(?<![A-Za-z0-9_])(?:bic|swift)(?:[A-Za-z0-9_\/]|[^\x00-\x7F])*\s*[:.\-]?\s*\(?\s*$/i],
|
|
90
119
|
// Postal code. A postal code loses every contested span on priority (it is
|
|
91
120
|
// the weakest evidence the engine has), so without a cue entry
|
|
92
121
|
// "Cod postal: 040171" was handed to a German phone pattern.
|
|
93
122
|
[EntityType.POSTAL_CODE,
|
|
94
|
-
/(?<![A-Za-z0-9_])(?:cod\s*po[sș]tal|postnummer|postnr|postleitzahl|plz
|
|
123
|
+
/(?<![A-Za-z0-9_])(?:cod\s*po[sș]tal|postnummer|postnr|postleitzahl|plz(?:-?bereiche?)?|postcode|code\s*postal|codice\s*postale|kod\s*pocztowy|ps[čc]|τ\.?κ\.?|пощенски\s*код)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
95
124
|
// Internal / employee identifier. The canon calls these INTERNAL_ID and
|
|
96
125
|
// annotates them with an LLM, so the engine has no pattern for them — but it
|
|
97
126
|
// does have a phone pattern that was happily claiming them. The cue is the
|
|
98
127
|
// only thing that identifies one, which is why the engine emits this type
|
|
99
128
|
// from here and nowhere else.
|
|
100
129
|
[EntityType.INTERNAL_ID,
|
|
101
|
-
/(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge
|
|
130
|
+
/(?<![A-Za-z0-9_])(?:medarbejdernummer|ansattnummer|anställningsnummer|personalnummer|personeelsnummer|matricule\s*salarié|employee\s*(?:no|number|id)|badge(?:\s*(?:no|number))?|betriebsstätten(?:nr|nummer)|grundstücks(?:nr|nummer)|dossier-?(?:nr|nummer)|aktenzeichen|osobní\s*(?:číslo\s*)?zaměstnance)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
131
|
+
// Credential. Typed SECRET because that is the type this SDK has; the
|
|
132
|
+
// training pipeline's report asks for CREDENTIAL, which does not exist here.
|
|
133
|
+
[EntityType.SECRET,
|
|
134
|
+
/(?<![A-Za-z0-9_])(?:tan-?activatiecode|activatiecode|activation\s*code|verificatiecode|verification\s*code|otp|pincode)(?:(?:[A-Za-z0-9_]|[^\x00-\x7F])*|\s+[^\s:.\-\d,;()\/]{2,20})\s*\)?\s*[:.\-]*\s*$/i],
|
|
102
135
|
];
|
|
103
136
|
/**
|
|
104
137
|
* The entity type a label immediately before `start` names, or null.
|
|
@@ -124,22 +157,56 @@ const RETYPABLE = new Set([
|
|
|
124
157
|
EntityType.PHONE, EntityType.POSTAL_CODE, EntityType.LICENSE_PLATE,
|
|
125
158
|
]);
|
|
126
159
|
/**
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
160
|
+
* Retypable, but only when the document gives its country no support at all.
|
|
161
|
+
*
|
|
162
|
+
* These carry a checksum, so normally the pattern outranks any label. But a
|
|
163
|
+
* checksum only says the digits fit *some* national scheme, and a weak one fits
|
|
164
|
+
* by luck — the same reasoning that demotes an uncorroborated validated
|
|
165
|
+
* candidate to priority 1 in `engine.ts`. At a country score of 0.0 the engine
|
|
166
|
+
* is already saying "this passed a Czech checksum in a document with nothing
|
|
167
|
+
* Czech about it", and an explicit "Passport No.:" is the better evidence.
|
|
168
|
+
*
|
|
169
|
+
* Restricted to zero country support on purpose: a Dutch BSN in a Dutch
|
|
170
|
+
* document scores above zero and stays a NATIONAL_ID whatever label sits near
|
|
171
|
+
* it, so this cannot relabel a domestic identifier — only the coincidental
|
|
172
|
+
* foreign ones, which is where "Passport No.: 512847603" and
|
|
173
|
+
* "Betriebsstättennr. 260326822" both landed, each reported as Czech.
|
|
174
|
+
*/
|
|
175
|
+
const RETYPABLE_UNCORROBORATED = new Set([EntityType.NATIONAL_ID]);
|
|
176
|
+
/**
|
|
177
|
+
* Types a label can assert on its own.
|
|
130
178
|
*
|
|
131
179
|
* PHONE is absent on purpose: "Tel:" in front of something no phone pattern
|
|
132
180
|
* matched means the document is laid out oddly, not that the value is a phone
|
|
133
|
-
* number. So are EMAIL,
|
|
181
|
+
* number. So are EMAIL, DOB, VIN and the rest — those carry their own
|
|
134
182
|
* structure, and a nearby word is not entitled to overrule it.
|
|
183
|
+
*/
|
|
184
|
+
export const CUE_TARGETS = new Set([
|
|
185
|
+
EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID,
|
|
186
|
+
EntityType.HEALTH_INSURANCE, EntityType.HEALTHCARE_PROVIDER,
|
|
187
|
+
EntityType.CHAMBER_OF_COMMERCE, EntityType.VAT, EntityType.POSTAL_CODE,
|
|
188
|
+
EntityType.PASSPORT, EntityType.INTERNAL_ID, EntityType.BANK_ACCOUNT,
|
|
189
|
+
EntityType.SECRET,
|
|
190
|
+
]);
|
|
191
|
+
/**
|
|
192
|
+
* Which of those a label may also *rescue* — re-admit after its checksum
|
|
193
|
+
* failed. Until 0.3.9 this was the same set as CUE_TARGETS and it cannot stay
|
|
194
|
+
* that way: re-typing a span the engine already decided to mask is cheap to be
|
|
195
|
+
* wrong about, while rescuing one asserts that a value which failed its check
|
|
196
|
+
* digit is a real identifier anyway.
|
|
135
197
|
*
|
|
136
198
|
* BIC's absence is what keeps the rescue honest. Its validator is a *registry
|
|
137
199
|
* lookup*, not a checksum, so a failure means "no such bank", which no label can
|
|
138
200
|
* talk you out of. Rescuing it read 34 unlisted codes behind a "BIC:" label as
|
|
139
201
|
* real ones. The same reasoning keeps CREDIT_CARD and BANK_ACCOUNT out: Luhn and
|
|
140
202
|
* mod-97 are strong enough that a failure really does mean "not one of these".
|
|
203
|
+
* BANK_ACCOUNT joined CUE_TARGETS in 0.3.9 so "sort code 20-45-91" stops being a
|
|
204
|
+
* LICENSE_PLATE, and is kept out here for exactly the 0.3.8 reason.
|
|
205
|
+
*
|
|
206
|
+
* HEALTHCARE_PROVIDER and SECRET are absent because neither is checksummed;
|
|
207
|
+
* there is no failed validation for a label to overrule in the first place.
|
|
141
208
|
*/
|
|
142
|
-
export const
|
|
209
|
+
export const RESCUE_TARGETS = new Set([
|
|
143
210
|
EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID,
|
|
144
211
|
EntityType.HEALTH_INSURANCE, EntityType.CHAMBER_OF_COMMERCE,
|
|
145
212
|
EntityType.VAT, EntityType.POSTAL_CODE, EntityType.PASSPORT,
|
|
@@ -152,9 +219,14 @@ export const CUE_TARGETS = new Set([
|
|
|
152
219
|
* without them fired 52 times and got roughly 30 of those wrong — EMAIL read as
|
|
153
220
|
* VAT, a date of birth as a phone number, a VIN as VAT — against 20 real fixes.
|
|
154
221
|
* With them: 14 re-typings, every one of them PHONE to a structured identifier.
|
|
222
|
+
*
|
|
223
|
+
* `countryScore` is the document's support for the country the winning pattern
|
|
224
|
+
* came from; see RETYPABLE_UNCORROBORATED.
|
|
155
225
|
*/
|
|
156
|
-
export function retypedBy(text, start, entityType) {
|
|
157
|
-
|
|
226
|
+
export function retypedBy(text, start, entityType, countryScore) {
|
|
227
|
+
const type = String(entityType);
|
|
228
|
+
if (!RETYPABLE.has(type)
|
|
229
|
+
&& !(RETYPABLE_UNCORROBORATED.has(type) && countryScore === 0))
|
|
158
230
|
return null;
|
|
159
231
|
const cued = cuedType(text, start);
|
|
160
232
|
if (cued === null || cued === entityType || !CUE_TARGETS.has(String(cued)))
|
|
@@ -8,6 +8,38 @@ import { type RawMatch } from "./matchers.js";
|
|
|
8
8
|
export declare const COUNTRY_CODE_ALIASES: Record<string, string>;
|
|
9
9
|
/** Map a caller-supplied code to a registered one, or null if unknown. */
|
|
10
10
|
export declare function resolveCountryCode(countryCode: string): string | null;
|
|
11
|
+
/**
|
|
12
|
+
* Last-resort ordering for candidates every other field has tied on.
|
|
13
|
+
*
|
|
14
|
+
* Reaching this means the engine has no evidence left: same span, no cue, same
|
|
15
|
+
* priority, the same (usually zero) country score, and the same declared scope.
|
|
16
|
+
* Something still has to win, and before this the winner was whichever candidate
|
|
17
|
+
* happened to be generated first — which is country *registration* order, and
|
|
18
|
+
* the two SDKs do not share one. Python registers alphabetically
|
|
19
|
+
* (`AT, BE, BG, CH, CY, CZ, ...`); this SDK uses a curated order
|
|
20
|
+
* (`NL, BE, DE, AT, CH, FR, ..., PT, LU, PL, ..., CZ`). So a bare digit run that
|
|
21
|
+
* several national schemes accept was filed as a Czech `NATIONAL_ID` by Python
|
|
22
|
+
* and a Portuguese `PHONE` here — identical characters masked, contradictory
|
|
23
|
+
* labels, and `make parity` compared only characters, so it stayed green through
|
|
24
|
+
* the whole of 0.3.8.
|
|
25
|
+
*
|
|
26
|
+
* The order reproduces what Python already did rather than inventing a new one,
|
|
27
|
+
* because every accuracy figure this engine has published was measured with
|
|
28
|
+
* Python resolving these ties by its registration order. Ordering on the entity
|
|
29
|
+
* type instead was tried first and re-tuned the corpus twice over: it turned the
|
|
30
|
+
* German `Steuerß12345678911` from `TAX_ID`/DE into `NATIONAL_ID`/EL, and moved
|
|
31
|
+
* 820 further spans from `DOB` to `DATE_OF_DEATH`. So the key is deliberately
|
|
32
|
+
* not total — candidates from the same country are left to the stable sort,
|
|
33
|
+
* which preserves the order that country's pattern list was written in, and both
|
|
34
|
+
* SDKs write theirs in the same order.
|
|
35
|
+
*
|
|
36
|
+
* Taken from the **pattern**, never the finished detection: this SDK re-types a
|
|
37
|
+
* cued candidate before the sort and Python re-types after it, so keying on the
|
|
38
|
+
* final type would reintroduce the divergence in a subtler form.
|
|
39
|
+
*
|
|
40
|
+
* Kept character-for-character in step with `tie_key` in `rules/engine.py`.
|
|
41
|
+
*/
|
|
42
|
+
export declare function tieKey(countryCode: string): string;
|
|
11
43
|
export declare class RuleEngine {
|
|
12
44
|
private matcher;
|
|
13
45
|
private loadedCountries;
|
package/dist/esm/rules/engine.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { DetectionSource, EntityType } from "../types.js";
|
|
2
2
|
import { MultiPatternMatcher } from "./matchers.js";
|
|
3
3
|
import { shouldSuppress, SuppressionScratch } from "./suppressors.js";
|
|
4
|
-
import {
|
|
4
|
+
import { RESCUE_TARGETS, cuedType, retypedBy } from "./cues.js";
|
|
5
5
|
import { detectStructuralDob } from "./structural.js";
|
|
6
6
|
import { COUNTRY_CONFIGS } from "./countries/index.js";
|
|
7
7
|
import * as evidenceMod from "./evidence.js";
|
|
@@ -44,6 +44,48 @@ export function resolveCountryCode(countryCode) {
|
|
|
44
44
|
code = COUNTRY_CODE_ALIASES[code] ?? code;
|
|
45
45
|
return code in COUNTRY_CONFIGS && code !== "SHARED" ? code : null;
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* Last-resort ordering for candidates every other field has tied on.
|
|
49
|
+
*
|
|
50
|
+
* Reaching this means the engine has no evidence left: same span, no cue, same
|
|
51
|
+
* priority, the same (usually zero) country score, and the same declared scope.
|
|
52
|
+
* Something still has to win, and before this the winner was whichever candidate
|
|
53
|
+
* happened to be generated first — which is country *registration* order, and
|
|
54
|
+
* the two SDKs do not share one. Python registers alphabetically
|
|
55
|
+
* (`AT, BE, BG, CH, CY, CZ, ...`); this SDK uses a curated order
|
|
56
|
+
* (`NL, BE, DE, AT, CH, FR, ..., PT, LU, PL, ..., CZ`). So a bare digit run that
|
|
57
|
+
* several national schemes accept was filed as a Czech `NATIONAL_ID` by Python
|
|
58
|
+
* and a Portuguese `PHONE` here — identical characters masked, contradictory
|
|
59
|
+
* labels, and `make parity` compared only characters, so it stayed green through
|
|
60
|
+
* the whole of 0.3.8.
|
|
61
|
+
*
|
|
62
|
+
* The order reproduces what Python already did rather than inventing a new one,
|
|
63
|
+
* because every accuracy figure this engine has published was measured with
|
|
64
|
+
* Python resolving these ties by its registration order. Ordering on the entity
|
|
65
|
+
* type instead was tried first and re-tuned the corpus twice over: it turned the
|
|
66
|
+
* German `Steuerß12345678911` from `TAX_ID`/DE into `NATIONAL_ID`/EL, and moved
|
|
67
|
+
* 820 further spans from `DOB` to `DATE_OF_DEATH`. So the key is deliberately
|
|
68
|
+
* not total — candidates from the same country are left to the stable sort,
|
|
69
|
+
* which preserves the order that country's pattern list was written in, and both
|
|
70
|
+
* SDKs write theirs in the same order.
|
|
71
|
+
*
|
|
72
|
+
* Taken from the **pattern**, never the finished detection: this SDK re-types a
|
|
73
|
+
* cued candidate before the sort and Python re-types after it, so keying on the
|
|
74
|
+
* final type would reintroduce the divergence in a subtler form.
|
|
75
|
+
*
|
|
76
|
+
* Kept character-for-character in step with `tie_key` in `rules/engine.py`.
|
|
77
|
+
*/
|
|
78
|
+
export function tieKey(countryCode) {
|
|
79
|
+
// "SHARED" and "CUSTOM" are not countries and sort as themselves under a
|
|
80
|
+
// plain alphabetical key — landing between PT and SK, rather than ahead of
|
|
81
|
+
// every country the way the registry loads them. The bucket restores that.
|
|
82
|
+
const bucket = countryCode === "SHARED" ? "0" : countryCode === "CUSTOM" ? "1" : "2";
|
|
83
|
+
// NUL separator, written as an escape so this file stays plain ASCII, and
|
|
84
|
+
// matching Python byte for byte. It must sort below every character a code
|
|
85
|
+
// or type name can contain, or a code that is a prefix of another would
|
|
86
|
+
// order differently in the two SDKs.
|
|
87
|
+
return `${bucket}\u0000${countryCode}`;
|
|
88
|
+
}
|
|
47
89
|
const warnedCountries = new Set();
|
|
48
90
|
function warnUnknownCountry(code) {
|
|
49
91
|
if (warnedCountries.has(code))
|
|
@@ -203,7 +245,7 @@ export class RuleEngine {
|
|
|
203
245
|
else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
|
|
204
246
|
failedSpans.push({ start: m.start, end: m.end, code: m.countryCode,
|
|
205
247
|
etype: String(m.patternDef.entityType) });
|
|
206
|
-
if (
|
|
248
|
+
if (RESCUE_TARGETS.has(String(m.patternDef.entityType))
|
|
207
249
|
&& cuedType(text, m.start) === m.patternDef.entityType)
|
|
208
250
|
rescued.push(m);
|
|
209
251
|
}
|
|
@@ -415,7 +457,7 @@ export class RuleEngine {
|
|
|
415
457
|
// the value is filed as a Cypriot identity number. Dropping it also clears
|
|
416
458
|
// outOfScope, which would otherwise claim the detection belongs to a
|
|
417
459
|
// country the caller did not declare while naming no country at all.
|
|
418
|
-
const retyped = retypedBy(text, match.start, entityType);
|
|
460
|
+
const retyped = retypedBy(text, match.start, entityType, countryScores.get(match.countryCode) ?? 0);
|
|
419
461
|
if (retyped !== null) {
|
|
420
462
|
entityType = retyped;
|
|
421
463
|
country = null;
|
|
@@ -429,6 +471,7 @@ export class RuleEngine {
|
|
|
429
471
|
length: match.end - match.start,
|
|
430
472
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
431
473
|
inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
|
|
474
|
+
tie: tieKey(match.countryCode),
|
|
432
475
|
tier: 0,
|
|
433
476
|
det: {
|
|
434
477
|
entityType,
|
|
@@ -467,6 +510,7 @@ export class RuleEngine {
|
|
|
467
510
|
length: match.end - match.start,
|
|
468
511
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
469
512
|
inScope: declared === null || declared.has(match.countryCode) ? 1 : 0,
|
|
513
|
+
tie: tieKey(match.countryCode),
|
|
470
514
|
tier: 0,
|
|
471
515
|
det: {
|
|
472
516
|
entityType: match.patternDef.entityType,
|
|
@@ -482,7 +526,8 @@ export class RuleEngine {
|
|
|
482
526
|
}
|
|
483
527
|
for (const d of detectStructuralDob(text)) {
|
|
484
528
|
candidates.push({ blindPriority: 1, cue: 0, priority: 1, length: d.end - d.start,
|
|
485
|
-
countryScore: 0, inScope: 1, tier: 0, det: d
|
|
529
|
+
countryScore: 0, inScope: 1, tier: 0, det: d,
|
|
530
|
+
tie: tieKey(d.country ?? "") });
|
|
486
531
|
}
|
|
487
532
|
const detections = this.deduplicate(candidates);
|
|
488
533
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
@@ -550,7 +595,8 @@ export class RuleEngine {
|
|
|
550
595
|
b.cue - a.cue || // local cue } country-aware
|
|
551
596
|
b.priority - a.priority || // country priority } — decides the
|
|
552
597
|
b.countryScore - a.countryScore || // country evidence } LABEL only
|
|
553
|
-
b.inScope - a.inScope
|
|
598
|
+
b.inScope - a.inScope || // declared scope }
|
|
599
|
+
(a.tie < b.tie ? -1 : a.tie > b.tie ? 1 : 0)); // total } — see tieKey
|
|
554
600
|
const result = [];
|
|
555
601
|
const occupied = new Set();
|
|
556
602
|
for (const { det } of sorted) {
|
|
@@ -343,6 +343,53 @@ function suppressYearAsPostal(text, match) {
|
|
|
343
343
|
// Suppress: recent years without postal context are almost never postal codes
|
|
344
344
|
return true;
|
|
345
345
|
}
|
|
346
|
+
// ── Postal code: disqualified by the word in front of it ───────────────
|
|
347
|
+
//
|
|
348
|
+
// The mirror image of `rules/cues.ts`. There a label promotes a type; here a
|
|
349
|
+
// word refuses one. Both were reported together: a bare four-digit run becomes
|
|
350
|
+
// a POSTAL_CODE as soon as *any* real postal code establishes the country,
|
|
351
|
+
// which is to say in almost every real document, because POSTAL_CONTEXT_NEAR
|
|
352
|
+
// above then rescues it from suppressYearAsPostal. Measured on the training
|
|
353
|
+
// corpus: 55 bare years in prose ("Opgericht in 2016", "Fondée en 2017") plus
|
|
354
|
+
// telephone extensions ("(toest. 3841)", "(poste interne 3318)", "(ext. 2219)").
|
|
355
|
+
/**
|
|
356
|
+
* Prepositions that can only be temporal. Nothing is ever located "since 2018",
|
|
357
|
+
* so these disqualify a postal code on their own.
|
|
358
|
+
*/
|
|
359
|
+
const YEAR_WORD_BEFORE = /(?<![A-Za-z0-9_])(?:sinds|since|depuis|seit|siden|sedan|desde|vuonna|anno|dal)\s+$/i;
|
|
360
|
+
/**
|
|
361
|
+
* "in", "en" and "im" are *both* temporal and locative, and in exactly the
|
|
362
|
+
* countries whose postal codes are year-shaped. Belgian 2000 is Antwerp and
|
|
363
|
+
* 2018 is one of its districts, so "Rustige ligging in 2018, vlakbij openbaar
|
|
364
|
+
* vervoer" is an address, not a date — an earlier version of this rule
|
|
365
|
+
* suppressed it and lost a real postal code, which is the worse error for a
|
|
366
|
+
* redaction tool. So an ambiguous preposition needs a founding or payment
|
|
367
|
+
* participle in front of it, which is what every case in the report had:
|
|
368
|
+
* "Opgericht in 2016", "Fondée en 2017", "versé en 2025".
|
|
369
|
+
*/
|
|
370
|
+
const YEAR_VERB_BEFORE = /(?<![A-Za-z0-9_])(?:opgericht|gesticht|opgestart|founded|established|created|fond[ée]e?|cr[ée][ée]e?|gegründet|gegruendet|errichtet|grundlagt|grundad|perustettu|fundada|fundado|fondata|costituita|vers[ée]|betaald|uitbetaald|ausgezahlt|paid|geboren|born)\s+(?:in|en|im|op|the)\s+$/i;
|
|
371
|
+
/**
|
|
372
|
+
* A capitalised word immediately after keeps the candidate: "in 2000 Antwerpen"
|
|
373
|
+
* really is a postal code, and so is the "AZ" of a Dutch "1105 AZ". Without
|
|
374
|
+
* this the rule would suppress the very addresses it is meant to leave alone.
|
|
375
|
+
*/
|
|
376
|
+
const TOWN_AFTER = /^\s*[A-ZÀ-ÞĀ-Ž]/;
|
|
377
|
+
/**
|
|
378
|
+
* A telephone extension marker. "poste interne 3318" puts one qualifier word
|
|
379
|
+
* between the marker and the number, so allow exactly one.
|
|
380
|
+
*/
|
|
381
|
+
const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|durchwahl|doorkiesnummer|tst|nebenstelle)\.?\s*(?:[a-zà-ÿ]{2,10}\s*)?[:.]?\s*$/i;
|
|
382
|
+
/** Suppress a postal code the word in front of it rules out. */
|
|
383
|
+
function suppressPostalAfterDisqualifyingWord(text, match) {
|
|
384
|
+
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
385
|
+
return false;
|
|
386
|
+
const before = text.slice(Math.max(0, match.start - 24), match.start);
|
|
387
|
+
if (EXTENSION_BEFORE.test(before))
|
|
388
|
+
return true;
|
|
389
|
+
return RECENT_YEAR.test(match.text.trim())
|
|
390
|
+
&& (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
|
|
391
|
+
&& !TOWN_AFTER.test(text.slice(match.end, match.end + 3));
|
|
392
|
+
}
|
|
346
393
|
function suppressPhoneServiceNumber(_text, match) {
|
|
347
394
|
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
348
395
|
return false;
|
|
@@ -791,7 +838,7 @@ const TYPE_SUPPRESSORS = {
|
|
|
791
838
|
[EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
792
839
|
[EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
793
840
|
[EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressTaxidAsIpAddress],
|
|
794
|
-
[EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier, suppressReference],
|
|
841
|
+
[EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalAfterDisqualifyingWord, suppressPostalInsideIban, suppressPostalAsHouseNumber, suppressPostalInLongerIdentifier, suppressReference],
|
|
795
842
|
[EntityType.BIC]: [suppressBicWithoutEvidence],
|
|
796
843
|
[EntityType.BANK_ACCOUNT]: [suppressReference],
|
|
797
844
|
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound, suppressDePlateUnknownDistrict, suppressPlateAsCurrencyAmount],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.9",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
5
|
"homepage": "https://euredact.dev",
|
|
6
6
|
"license": "Apache-2.0",
|