euredact 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/rules/context.js +14 -2
- package/dist/cjs/rules/countries/index.js +1 -1
- package/dist/cjs/rules/engine.js +87 -11
- package/dist/cjs/sdk.js +29 -1
- package/dist/esm/rules/context.d.ts +14 -2
- package/dist/esm/rules/context.js +14 -2
- package/dist/esm/rules/countries/index.js +1 -1
- package/dist/esm/rules/engine.js +87 -11
- package/dist/esm/sdk.js +29 -1
- package/package.json +3 -2
|
@@ -39,8 +39,20 @@ class DocumentContext {
|
|
|
39
39
|
this.items.push(chunkOffset === 0 ? ev : { ...ev, span });
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
|
-
/**
|
|
43
|
-
|
|
42
|
+
/**
|
|
43
|
+
* Everything recorded so far, spans rebased to the whole document.
|
|
44
|
+
*
|
|
45
|
+
* A getter, to match {@link DocumentContext.size}. These were a method and a
|
|
46
|
+
* getter on the same class, which is a trip hazard rather than a style
|
|
47
|
+
* question: `ctx.size()` and `ctx.evidence` both *succeed* and return
|
|
48
|
+
* something useless — a number is not callable, and a function object is
|
|
49
|
+
* truthy — so the mistake surfaces later as an empty result rather than as
|
|
50
|
+
* an error. Both are property reads now, and the wrong form throws.
|
|
51
|
+
*
|
|
52
|
+
* Returns a fresh copy each call, so mutating it cannot corrupt the context.
|
|
53
|
+
* (The Python SDK spells this `evidence()`, following its own idiom.)
|
|
54
|
+
*/
|
|
55
|
+
get evidence() {
|
|
44
56
|
return [...this.items];
|
|
45
57
|
}
|
|
46
58
|
get size() {
|
|
@@ -226,7 +226,7 @@ const DE = {
|
|
|
226
226
|
p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b[A-ZÄÖÜ]{1,3}[\s\-][A-Z]{1,2}[\s\-]?\d{1,4}[EH]?\b`),
|
|
227
227
|
p(types_js_1.EntityType.POSTAL_CODE, String.raw `\b\d{5}\b`, null, "", ["PLZ", "Postleitzahl", "postal code", "postcode", "Anschrift", "Adresse", "Wohnort", "Ort", "Stadt", "wohnt", "wohnhaft", "Straße", "Str.", "Weg", "Platz", "Allee", "Ring"], true),
|
|
228
228
|
p(types_js_1.EntityType.CHAMBER_OF_COMMERCE, String.raw `\bHR[AB]\s?\d{4,6}\b`),
|
|
229
|
-
p(types_js_1.EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer"], true),
|
|
229
|
+
p(types_js_1.EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer", "SVNR", "SV-Nr", "RVNR"], true),
|
|
230
230
|
p(types_js_1.EntityType.TAX_ID, String.raw `\b\d{2,3}/\d{3,4}/\d{4,5}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
231
231
|
p(types_js_1.EntityType.TAX_ID, String.raw `\b\d{13}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
232
232
|
p(types_js_1.EntityType.HEALTH_INSURANCE, String.raw `\b[A-Z]\d{9}\b`, null, "", ["Versichertennummer", "Krankenversicherung", "KV-Nummer", "Krankenkasse", "GKV", "Versichertenkarte", "Versicherter", "Versicherte", "KVNR", "Krankenversichertennummer"], true),
|
package/dist/cjs/rules/engine.js
CHANGED
|
@@ -253,6 +253,23 @@ class RuleEngine {
|
|
|
253
253
|
else
|
|
254
254
|
zonesByType.set(f.etype, [[f.start, f.end]]);
|
|
255
255
|
}
|
|
256
|
+
// Any type, strict subset: a candidate covering only *part* of a rejected
|
|
257
|
+
// identifier is a fragment of it, not a different entity. The generic
|
|
258
|
+
// separator-tolerant phone pattern claimed characters 3-14 of the rejected
|
|
259
|
+
// Belgian national number in "Rijksregisternummer: 85.03.19-284.73" and
|
|
260
|
+
// reported it as a PHONE. An equal span is two schemes competing for the
|
|
261
|
+
// whole value; a strict subset is one picking over another's wreckage.
|
|
262
|
+
//
|
|
263
|
+
// Deliberately ignores corroboration and uses every failed span, because it
|
|
264
|
+
// *removes* candidates and must therefore be country-blind — gating it on
|
|
265
|
+
// the document's countries would make the set of spans found depend on
|
|
266
|
+
// `countries`, which is invariant I1.
|
|
267
|
+
//
|
|
268
|
+
// Kept unmerged: merging adjacent spans would invent strict-subset
|
|
269
|
+
// relationships that neither original span had.
|
|
270
|
+
const allFailed = failedSpans
|
|
271
|
+
.map(f => [f.start, f.end])
|
|
272
|
+
.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
256
273
|
const mergedZones = new Map();
|
|
257
274
|
for (const [etype, spans] of zonesByType) {
|
|
258
275
|
spans.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
@@ -299,9 +316,28 @@ class RuleEngine {
|
|
|
299
316
|
demoted = idx >= 0 && zone.ends[idx] >= match.end;
|
|
300
317
|
}
|
|
301
318
|
}
|
|
319
|
+
// Dropped rather than demoted. Demotion cannot help here: nothing else
|
|
320
|
+
// wants a fragment, so a demoted candidate still wins its span by
|
|
321
|
+
// default — the generic phone pattern kept claiming the tail of the
|
|
322
|
+
// rejected national number.
|
|
323
|
+
if (match.patternDef.validator === null && !demoted) {
|
|
324
|
+
let fragment = false;
|
|
325
|
+
for (const [zs, ze] of allFailed) {
|
|
326
|
+
if (zs > match.start)
|
|
327
|
+
break;
|
|
328
|
+
if (match.end <= ze && (match.start > zs || match.end < ze)) {
|
|
329
|
+
fragment = true;
|
|
330
|
+
break;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
if (fragment)
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
302
336
|
let priority;
|
|
337
|
+
let blindPriority;
|
|
303
338
|
if (demoted) {
|
|
304
339
|
priority = -1;
|
|
340
|
+
blindPriority = -1;
|
|
305
341
|
}
|
|
306
342
|
else if (hasValidValidator) {
|
|
307
343
|
// A passing checksum normally outranks everything. But a checksum only
|
|
@@ -315,9 +351,11 @@ class RuleEngine {
|
|
|
315
351
|
// themselves — an IBAN emits evidence for its own country — so a
|
|
316
352
|
// foreign IBAN in a domestic invoice keeps its rank.
|
|
317
353
|
priority = corroborated(match.countryCode) ? 3 : 1;
|
|
354
|
+
blindPriority = 3;
|
|
318
355
|
}
|
|
319
356
|
else if (match.countryCode === "CUSTOM") {
|
|
320
357
|
priority = 2;
|
|
358
|
+
blindPriority = 2;
|
|
321
359
|
}
|
|
322
360
|
else if (match.patternDef.entityType === types_js_1.EntityType.POSTAL_CODE) {
|
|
323
361
|
// A bare digit run is the weakest evidence in the engine and must never
|
|
@@ -325,13 +363,16 @@ class RuleEngine {
|
|
|
325
363
|
// BANK_ACCOUNT, VAT...). Postal codes take unclaimed spans only,
|
|
326
364
|
// whatever the span lengths.
|
|
327
365
|
priority = 0;
|
|
366
|
+
blindPriority = 0;
|
|
328
367
|
}
|
|
329
368
|
else {
|
|
330
369
|
priority = 1;
|
|
370
|
+
blindPriority = 1;
|
|
331
371
|
}
|
|
332
372
|
const country = (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
|
|
333
373
|
? match.countryCode : null;
|
|
334
374
|
candidates.push({
|
|
375
|
+
blindPriority,
|
|
335
376
|
priority,
|
|
336
377
|
length: match.end - match.start,
|
|
337
378
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
@@ -349,7 +390,8 @@ class RuleEngine {
|
|
|
349
390
|
});
|
|
350
391
|
}
|
|
351
392
|
for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
|
|
352
|
-
candidates.push({ priority: 1, length: d.end - d.start,
|
|
393
|
+
candidates.push({ blindPriority: 1, priority: 1, length: d.end - d.start,
|
|
394
|
+
countryScore: 0, inScope: 1, det: d });
|
|
353
395
|
}
|
|
354
396
|
const detections = this.deduplicate(candidates);
|
|
355
397
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
@@ -370,16 +412,50 @@ class RuleEngine {
|
|
|
370
412
|
deduplicate(candidates) {
|
|
371
413
|
if (candidates.length === 0)
|
|
372
414
|
return [];
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
//
|
|
379
|
-
const
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
415
|
+
// The structural tier is a property of the *span*, not of the candidate:
|
|
416
|
+
// the best tier anything claiming those exact characters can offer. Used
|
|
417
|
+
// per candidate it would also discriminate *within* a span, where the
|
|
418
|
+
// country-aware tier must have the final word — that silently disabled
|
|
419
|
+
// country inference entirely in the Python SDK, handing every ambiguous
|
|
420
|
+
// value to whichever foreign scheme happened to validate.
|
|
421
|
+
const spanTier = new Map();
|
|
422
|
+
for (const c of candidates) {
|
|
423
|
+
const key = `${c.det.start}:${c.det.end}`;
|
|
424
|
+
const seen = spanTier.get(key);
|
|
425
|
+
if (seen === undefined || c.blindPriority > seen)
|
|
426
|
+
spanTier.set(key, c.blindPriority);
|
|
427
|
+
}
|
|
428
|
+
const tierOf = (c) => spanTier.get(`${c.det.start}:${c.det.end}`);
|
|
429
|
+
// Span length outranks *everything*, including validation.
|
|
430
|
+
//
|
|
431
|
+
// A shorter candidate winning re-cuts a longer one and leaves the
|
|
432
|
+
// remainder in the clear, which is a leak. Two measured cases:
|
|
433
|
+
//
|
|
434
|
+
// "06 12 34 56 78" the Belgian phone pattern claimed 11 of 14
|
|
435
|
+
// characters and exposed three digits
|
|
436
|
+
// "194.232.104.77" the Dutch national-ID pattern validates on
|
|
437
|
+
// "194.232.104" and outranked the IP address,
|
|
438
|
+
// leaving ".77" in the output
|
|
439
|
+
//
|
|
440
|
+
// The second also broke invariant I1 in the Python SDK, where promotion
|
|
441
|
+
// depends on whether the document corroborates the country, so `countries`
|
|
442
|
+
// decided which span won. Ranking length first removes both — a redaction
|
|
443
|
+
// tool masking more than it must is recoverable; masking less is not.
|
|
444
|
+
//
|
|
445
|
+
// Priority still decides between candidates of *equal* length, which is
|
|
446
|
+
// where it matters: a valid IBAN beats a coincidental phone match on the
|
|
447
|
+
// same characters.
|
|
448
|
+
// Ordering between *different* spans is country-blind; the country only
|
|
449
|
+
// ever picks the winner among candidates covering the *same* span. That
|
|
450
|
+
// split makes invariant I1 structural rather than merely tested: the key is
|
|
451
|
+
// country-blind up to and including `start`, and (length, start) already
|
|
452
|
+
// identifies a span uniquely.
|
|
453
|
+
const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first } country-blind
|
|
454
|
+
tierOf(b) - tierOf(a) || // best tier here } — decides
|
|
455
|
+
a.det.start - b.det.start || // leftmost } WHICH span
|
|
456
|
+
b.priority - a.priority || // country priority } country-aware
|
|
457
|
+
b.countryScore - a.countryScore || // country evidence } — decides the
|
|
458
|
+
b.inScope - a.inScope); // declared scope } LABEL only
|
|
383
459
|
const result = [];
|
|
384
460
|
const occupied = new Set();
|
|
385
461
|
for (const { det } of sorted) {
|
package/dist/cjs/sdk.js
CHANGED
|
@@ -24,6 +24,32 @@ class ReferentialMapper {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
|
|
27
|
+
/**
|
|
28
|
+
* Reject a bare string where a list of country codes is expected.
|
|
29
|
+
*
|
|
30
|
+
* `countries: "NL"` is iterable, so it walks into the codes "N" and "L".
|
|
31
|
+
* Neither resolves, so nothing is declared — and every detection that *does*
|
|
32
|
+
* carry a country is then flagged `outOfScope`. A caller following the
|
|
33
|
+
* documented pattern of filtering on that field silently keeps none of them,
|
|
34
|
+
* while `redactedText` still looks perfectly correct. The failure direction is
|
|
35
|
+
* "no PII here", from a one-character typo.
|
|
36
|
+
*
|
|
37
|
+
* This already threw in Node, but by accident and only once something happened
|
|
38
|
+
* to call `.map` on it — deep in the call, with a message naming neither the
|
|
39
|
+
* argument nor the fix.
|
|
40
|
+
*
|
|
41
|
+
* A wrong *code* is data and only warns; raising there would invite callers to
|
|
42
|
+
* wrap redaction in try/catch and skip it. A wrong *type* is a programming
|
|
43
|
+
* error with no correct interpretation to fall back on.
|
|
44
|
+
*/
|
|
45
|
+
function checkCountryArg(value, param) {
|
|
46
|
+
if (typeof value === "string") {
|
|
47
|
+
throw new TypeError(`${param} must be an array of country codes, not a bare string. ` +
|
|
48
|
+
`Pass ${param}: ["${value}"] rather than ${param}: "${value}" — a string ` +
|
|
49
|
+
`is iterated character by character, which silently declares nothing and ` +
|
|
50
|
+
`flags every detection outOfScope.`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
27
53
|
class EuRedact {
|
|
28
54
|
engine = new engine_js_1.RuleEngine();
|
|
29
55
|
cache = new cache_js_1.ResultCache();
|
|
@@ -43,6 +69,8 @@ class EuRedact {
|
|
|
43
69
|
this.referentialMapper.clear();
|
|
44
70
|
}
|
|
45
71
|
redact(text, options = {}) {
|
|
72
|
+
checkCountryArg(options.countries, "countries");
|
|
73
|
+
checkCountryArg(options.countryHint, "countryHint");
|
|
46
74
|
if (text.length > this.maxInputLength) {
|
|
47
75
|
throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
|
|
48
76
|
`(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
|
|
@@ -67,7 +95,7 @@ class EuRedact {
|
|
|
67
95
|
if (cached !== null)
|
|
68
96
|
return cached;
|
|
69
97
|
}
|
|
70
|
-
const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence
|
|
98
|
+
const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence : null);
|
|
71
99
|
if (context !== null)
|
|
72
100
|
context.add(evidence, chunkOffset);
|
|
73
101
|
let detections = rawDetections;
|
|
@@ -25,7 +25,19 @@ export declare class DocumentContext {
|
|
|
25
25
|
private items;
|
|
26
26
|
/** Record evidence from a chunk starting at `chunkOffset`. */
|
|
27
27
|
add(evidence: CountryEvidence[], chunkOffset?: number): void;
|
|
28
|
-
/**
|
|
29
|
-
|
|
28
|
+
/**
|
|
29
|
+
* Everything recorded so far, spans rebased to the whole document.
|
|
30
|
+
*
|
|
31
|
+
* A getter, to match {@link DocumentContext.size}. These were a method and a
|
|
32
|
+
* getter on the same class, which is a trip hazard rather than a style
|
|
33
|
+
* question: `ctx.size()` and `ctx.evidence` both *succeed* and return
|
|
34
|
+
* something useless — a number is not callable, and a function object is
|
|
35
|
+
* truthy — so the mistake surfaces later as an empty result rather than as
|
|
36
|
+
* an error. Both are property reads now, and the wrong form throws.
|
|
37
|
+
*
|
|
38
|
+
* Returns a fresh copy each call, so mutating it cannot corrupt the context.
|
|
39
|
+
* (The Python SDK spells this `evidence()`, following its own idiom.)
|
|
40
|
+
*/
|
|
41
|
+
get evidence(): CountryEvidence[];
|
|
30
42
|
get size(): number;
|
|
31
43
|
}
|
|
@@ -36,8 +36,20 @@ export class DocumentContext {
|
|
|
36
36
|
this.items.push(chunkOffset === 0 ? ev : { ...ev, span });
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
/**
|
|
40
|
-
|
|
39
|
+
/**
|
|
40
|
+
* Everything recorded so far, spans rebased to the whole document.
|
|
41
|
+
*
|
|
42
|
+
* A getter, to match {@link DocumentContext.size}. These were a method and a
|
|
43
|
+
* getter on the same class, which is a trip hazard rather than a style
|
|
44
|
+
* question: `ctx.size()` and `ctx.evidence` both *succeed* and return
|
|
45
|
+
* something useless — a number is not callable, and a function object is
|
|
46
|
+
* truthy — so the mistake surfaces later as an empty result rather than as
|
|
47
|
+
* an error. Both are property reads now, and the wrong form throws.
|
|
48
|
+
*
|
|
49
|
+
* Returns a fresh copy each call, so mutating it cannot corrupt the context.
|
|
50
|
+
* (The Python SDK spells this `evidence()`, following its own idiom.)
|
|
51
|
+
*/
|
|
52
|
+
get evidence() {
|
|
41
53
|
return [...this.items];
|
|
42
54
|
}
|
|
43
55
|
get size() {
|
|
@@ -223,7 +223,7 @@ const DE = {
|
|
|
223
223
|
p(EntityType.LICENSE_PLATE, String.raw `\b[A-ZÄÖÜ]{1,3}[\s\-][A-Z]{1,2}[\s\-]?\d{1,4}[EH]?\b`),
|
|
224
224
|
p(EntityType.POSTAL_CODE, String.raw `\b\d{5}\b`, null, "", ["PLZ", "Postleitzahl", "postal code", "postcode", "Anschrift", "Adresse", "Wohnort", "Ort", "Stadt", "wohnt", "wohnhaft", "Straße", "Str.", "Weg", "Platz", "Allee", "Ring"], true),
|
|
225
225
|
p(EntityType.CHAMBER_OF_COMMERCE, String.raw `\bHR[AB]\s?\d{4,6}\b`),
|
|
226
|
-
p(EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer"], true),
|
|
226
|
+
p(EntityType.SSN, String.raw `\b\d{2}[\s]?\d{6}[\s]?[A-Z][\s]?\d{3}\b`, null, "", ["Rentenversicherung", "RV-Nummer", "Sozialversicherung", "SV-Nummer", "Versicherungsnummer", "Rentenversicherungsnummer", "Sozialversicherungsnummer", "SVNR", "SV-Nr", "RVNR"], true),
|
|
227
227
|
p(EntityType.TAX_ID, String.raw `\b\d{2,3}/\d{3,4}/\d{4,5}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
228
228
|
p(EntityType.TAX_ID, String.raw `\b\d{13}\b`, null, "", ["Steuernummer", "St.-Nr", "StNr", "Finanzamt", "Steuer"], true),
|
|
229
229
|
p(EntityType.HEALTH_INSURANCE, String.raw `\b[A-Z]\d{9}\b`, null, "", ["Versichertennummer", "Krankenversicherung", "KV-Nummer", "Krankenkasse", "GKV", "Versichertenkarte", "Versicherter", "Versicherte", "KVNR", "Krankenversichertennummer"], true),
|
package/dist/esm/rules/engine.js
CHANGED
|
@@ -216,6 +216,23 @@ export class RuleEngine {
|
|
|
216
216
|
else
|
|
217
217
|
zonesByType.set(f.etype, [[f.start, f.end]]);
|
|
218
218
|
}
|
|
219
|
+
// Any type, strict subset: a candidate covering only *part* of a rejected
|
|
220
|
+
// identifier is a fragment of it, not a different entity. The generic
|
|
221
|
+
// separator-tolerant phone pattern claimed characters 3-14 of the rejected
|
|
222
|
+
// Belgian national number in "Rijksregisternummer: 85.03.19-284.73" and
|
|
223
|
+
// reported it as a PHONE. An equal span is two schemes competing for the
|
|
224
|
+
// whole value; a strict subset is one picking over another's wreckage.
|
|
225
|
+
//
|
|
226
|
+
// Deliberately ignores corroboration and uses every failed span, because it
|
|
227
|
+
// *removes* candidates and must therefore be country-blind — gating it on
|
|
228
|
+
// the document's countries would make the set of spans found depend on
|
|
229
|
+
// `countries`, which is invariant I1.
|
|
230
|
+
//
|
|
231
|
+
// Kept unmerged: merging adjacent spans would invent strict-subset
|
|
232
|
+
// relationships that neither original span had.
|
|
233
|
+
const allFailed = failedSpans
|
|
234
|
+
.map(f => [f.start, f.end])
|
|
235
|
+
.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
219
236
|
const mergedZones = new Map();
|
|
220
237
|
for (const [etype, spans] of zonesByType) {
|
|
221
238
|
spans.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
@@ -262,9 +279,28 @@ export class RuleEngine {
|
|
|
262
279
|
demoted = idx >= 0 && zone.ends[idx] >= match.end;
|
|
263
280
|
}
|
|
264
281
|
}
|
|
282
|
+
// Dropped rather than demoted. Demotion cannot help here: nothing else
|
|
283
|
+
// wants a fragment, so a demoted candidate still wins its span by
|
|
284
|
+
// default — the generic phone pattern kept claiming the tail of the
|
|
285
|
+
// rejected national number.
|
|
286
|
+
if (match.patternDef.validator === null && !demoted) {
|
|
287
|
+
let fragment = false;
|
|
288
|
+
for (const [zs, ze] of allFailed) {
|
|
289
|
+
if (zs > match.start)
|
|
290
|
+
break;
|
|
291
|
+
if (match.end <= ze && (match.start > zs || match.end < ze)) {
|
|
292
|
+
fragment = true;
|
|
293
|
+
break;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (fragment)
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
265
299
|
let priority;
|
|
300
|
+
let blindPriority;
|
|
266
301
|
if (demoted) {
|
|
267
302
|
priority = -1;
|
|
303
|
+
blindPriority = -1;
|
|
268
304
|
}
|
|
269
305
|
else if (hasValidValidator) {
|
|
270
306
|
// A passing checksum normally outranks everything. But a checksum only
|
|
@@ -278,9 +314,11 @@ export class RuleEngine {
|
|
|
278
314
|
// themselves — an IBAN emits evidence for its own country — so a
|
|
279
315
|
// foreign IBAN in a domestic invoice keeps its rank.
|
|
280
316
|
priority = corroborated(match.countryCode) ? 3 : 1;
|
|
317
|
+
blindPriority = 3;
|
|
281
318
|
}
|
|
282
319
|
else if (match.countryCode === "CUSTOM") {
|
|
283
320
|
priority = 2;
|
|
321
|
+
blindPriority = 2;
|
|
284
322
|
}
|
|
285
323
|
else if (match.patternDef.entityType === EntityType.POSTAL_CODE) {
|
|
286
324
|
// A bare digit run is the weakest evidence in the engine and must never
|
|
@@ -288,13 +326,16 @@ export class RuleEngine {
|
|
|
288
326
|
// BANK_ACCOUNT, VAT...). Postal codes take unclaimed spans only,
|
|
289
327
|
// whatever the span lengths.
|
|
290
328
|
priority = 0;
|
|
329
|
+
blindPriority = 0;
|
|
291
330
|
}
|
|
292
331
|
else {
|
|
293
332
|
priority = 1;
|
|
333
|
+
blindPriority = 1;
|
|
294
334
|
}
|
|
295
335
|
const country = (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM")
|
|
296
336
|
? match.countryCode : null;
|
|
297
337
|
candidates.push({
|
|
338
|
+
blindPriority,
|
|
298
339
|
priority,
|
|
299
340
|
length: match.end - match.start,
|
|
300
341
|
countryScore: countryScores.get(match.countryCode) ?? 0,
|
|
@@ -312,7 +353,8 @@ export class RuleEngine {
|
|
|
312
353
|
});
|
|
313
354
|
}
|
|
314
355
|
for (const d of detectStructuralDob(text)) {
|
|
315
|
-
candidates.push({ priority: 1, length: d.end - d.start,
|
|
356
|
+
candidates.push({ blindPriority: 1, priority: 1, length: d.end - d.start,
|
|
357
|
+
countryScore: 0, inScope: 1, det: d });
|
|
316
358
|
}
|
|
317
359
|
const detections = this.deduplicate(candidates);
|
|
318
360
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
@@ -333,16 +375,50 @@ export class RuleEngine {
|
|
|
333
375
|
deduplicate(candidates) {
|
|
334
376
|
if (candidates.length === 0)
|
|
335
377
|
return [];
|
|
336
|
-
//
|
|
337
|
-
//
|
|
338
|
-
//
|
|
339
|
-
//
|
|
340
|
-
//
|
|
341
|
-
//
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
378
|
+
// The structural tier is a property of the *span*, not of the candidate:
|
|
379
|
+
// the best tier anything claiming those exact characters can offer. Used
|
|
380
|
+
// per candidate it would also discriminate *within* a span, where the
|
|
381
|
+
// country-aware tier must have the final word — that silently disabled
|
|
382
|
+
// country inference entirely in the Python SDK, handing every ambiguous
|
|
383
|
+
// value to whichever foreign scheme happened to validate.
|
|
384
|
+
const spanTier = new Map();
|
|
385
|
+
for (const c of candidates) {
|
|
386
|
+
const key = `${c.det.start}:${c.det.end}`;
|
|
387
|
+
const seen = spanTier.get(key);
|
|
388
|
+
if (seen === undefined || c.blindPriority > seen)
|
|
389
|
+
spanTier.set(key, c.blindPriority);
|
|
390
|
+
}
|
|
391
|
+
const tierOf = (c) => spanTier.get(`${c.det.start}:${c.det.end}`);
|
|
392
|
+
// Span length outranks *everything*, including validation.
|
|
393
|
+
//
|
|
394
|
+
// A shorter candidate winning re-cuts a longer one and leaves the
|
|
395
|
+
// remainder in the clear, which is a leak. Two measured cases:
|
|
396
|
+
//
|
|
397
|
+
// "06 12 34 56 78" the Belgian phone pattern claimed 11 of 14
|
|
398
|
+
// characters and exposed three digits
|
|
399
|
+
// "194.232.104.77" the Dutch national-ID pattern validates on
|
|
400
|
+
// "194.232.104" and outranked the IP address,
|
|
401
|
+
// leaving ".77" in the output
|
|
402
|
+
//
|
|
403
|
+
// The second also broke invariant I1 in the Python SDK, where promotion
|
|
404
|
+
// depends on whether the document corroborates the country, so `countries`
|
|
405
|
+
// decided which span won. Ranking length first removes both — a redaction
|
|
406
|
+
// tool masking more than it must is recoverable; masking less is not.
|
|
407
|
+
//
|
|
408
|
+
// Priority still decides between candidates of *equal* length, which is
|
|
409
|
+
// where it matters: a valid IBAN beats a coincidental phone match on the
|
|
410
|
+
// same characters.
|
|
411
|
+
// Ordering between *different* spans is country-blind; the country only
|
|
412
|
+
// ever picks the winner among candidates covering the *same* span. That
|
|
413
|
+
// split makes invariant I1 structural rather than merely tested: the key is
|
|
414
|
+
// country-blind up to and including `start`, and (length, start) already
|
|
415
|
+
// identifies a span uniquely.
|
|
416
|
+
const sorted = [...candidates].sort((a, b) => b.length - a.length || // longest first } country-blind
|
|
417
|
+
tierOf(b) - tierOf(a) || // best tier here } — decides
|
|
418
|
+
a.det.start - b.det.start || // leftmost } WHICH span
|
|
419
|
+
b.priority - a.priority || // country priority } country-aware
|
|
420
|
+
b.countryScore - a.countryScore || // country evidence } — decides the
|
|
421
|
+
b.inScope - a.inScope); // declared scope } LABEL only
|
|
346
422
|
const result = [];
|
|
347
423
|
const occupied = new Set();
|
|
348
424
|
for (const { det } of sorted) {
|
package/dist/esm/sdk.js
CHANGED
|
@@ -21,6 +21,32 @@ class ReferentialMapper {
|
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
23
|
const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
|
|
24
|
+
/**
|
|
25
|
+
* Reject a bare string where a list of country codes is expected.
|
|
26
|
+
*
|
|
27
|
+
* `countries: "NL"` is iterable, so it walks into the codes "N" and "L".
|
|
28
|
+
* Neither resolves, so nothing is declared — and every detection that *does*
|
|
29
|
+
* carry a country is then flagged `outOfScope`. A caller following the
|
|
30
|
+
* documented pattern of filtering on that field silently keeps none of them,
|
|
31
|
+
* while `redactedText` still looks perfectly correct. The failure direction is
|
|
32
|
+
* "no PII here", from a one-character typo.
|
|
33
|
+
*
|
|
34
|
+
* This already threw in Node, but by accident and only once something happened
|
|
35
|
+
* to call `.map` on it — deep in the call, with a message naming neither the
|
|
36
|
+
* argument nor the fix.
|
|
37
|
+
*
|
|
38
|
+
* A wrong *code* is data and only warns; raising there would invite callers to
|
|
39
|
+
* wrap redaction in try/catch and skip it. A wrong *type* is a programming
|
|
40
|
+
* error with no correct interpretation to fall back on.
|
|
41
|
+
*/
|
|
42
|
+
function checkCountryArg(value, param) {
|
|
43
|
+
if (typeof value === "string") {
|
|
44
|
+
throw new TypeError(`${param} must be an array of country codes, not a bare string. ` +
|
|
45
|
+
`Pass ${param}: ["${value}"] rather than ${param}: "${value}" — a string ` +
|
|
46
|
+
`is iterated character by character, which silently declares nothing and ` +
|
|
47
|
+
`flags every detection outOfScope.`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
24
50
|
export class EuRedact {
|
|
25
51
|
engine = new RuleEngine();
|
|
26
52
|
cache = new ResultCache();
|
|
@@ -40,6 +66,8 @@ export class EuRedact {
|
|
|
40
66
|
this.referentialMapper.clear();
|
|
41
67
|
}
|
|
42
68
|
redact(text, options = {}) {
|
|
69
|
+
checkCountryArg(options.countries, "countries");
|
|
70
|
+
checkCountryArg(options.countryHint, "countryHint");
|
|
43
71
|
if (text.length > this.maxInputLength) {
|
|
44
72
|
throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
|
|
45
73
|
`(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
|
|
@@ -64,7 +92,7 @@ export class EuRedact {
|
|
|
64
92
|
if (cached !== null)
|
|
65
93
|
return cached;
|
|
66
94
|
}
|
|
67
|
-
const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence
|
|
95
|
+
const { detections: rawDetections, evidence, scores, } = this.engine.detectWithEvidence(normalizedText, countries, countryHint, context !== null ? context.evidence : null);
|
|
68
96
|
if (context !== null)
|
|
69
97
|
context.add(evidence, chunkOffset);
|
|
70
98
|
let detections = rawDetections;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.4",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -60,7 +60,8 @@
|
|
|
60
60
|
"test:eval": "node --import tsx src/__tests__/run.ts",
|
|
61
61
|
"test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts",
|
|
62
62
|
"test:conformance": "node --import tsx src/__tests__/conformance.ts",
|
|
63
|
-
"test:inference": "node --import tsx src/__tests__/inference.ts"
|
|
63
|
+
"test:inference": "node --import tsx src/__tests__/inference.ts",
|
|
64
|
+
"dump": "node --import tsx src/__tests__/dump.ts"
|
|
64
65
|
},
|
|
65
66
|
"devDependencies": {
|
|
66
67
|
"typescript": "^5.4.0",
|