euredact 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +455 -44
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +124 -14
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +83 -13
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +479 -26
- package/dist/cjs/rules/validators.js +50 -0
- package/dist/cjs/sdk.js +108 -14
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +83 -1
- package/dist/esm/cloud/client.js +119 -22
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +83 -13
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +478 -25
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +47 -0
- package/dist/esm/sdk.d.ts +77 -4
- package/dist/esm/sdk.js +108 -17
- package/dist/esm/types.d.ts +64 -6
- package/dist/esm/types.js +4 -1
- package/package.json +13 -4
|
@@ -65,7 +65,21 @@ export declare function validateSpanishDni(candidate: string): boolean;
|
|
|
65
65
|
export declare function validateSpanishNie(candidate: string): boolean;
|
|
66
66
|
export declare function validatePortugueseNif(candidate: string): boolean;
|
|
67
67
|
export declare function validatePolishPesel(candidate: string): boolean;
|
|
68
|
+
/** Polish domestic account number (NRB): the PL IBAN without its "PL", so the IBAN's mod-97 decides it (rules-engine#93). */
|
|
69
|
+
export declare function validatePolishNrb(candidate: string): boolean;
|
|
68
70
|
export declare function validatePolishNip(candidate: string): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Polish identity card (dowód osobisty): 3 letters + 6 digits, the first digit
|
|
73
|
+
* a check digit. Letters A=10 … Z=35, weights 7,3,1,9,7,3,1,7,3: the weighted
|
|
74
|
+
* sum of all nine is 0 mod 10 exactly when the check digit is right (rules-engine#75).
|
|
75
|
+
*/
|
|
76
|
+
export declare function validatePolishIdCard(candidate: string): boolean;
|
|
77
|
+
/**
|
|
78
|
+
* Polish REGON: 9 digits, or 14 for a local unit, each with a mod-11 check (a
|
|
79
|
+
* remainder of 10 counts as 0). A 14-digit REGON begins with its parent's valid
|
|
80
|
+
* 9-digit REGON (rules-engine#76).
|
|
81
|
+
*/
|
|
82
|
+
export declare function validatePolishRegon(candidate: string): boolean;
|
|
69
83
|
export declare function validateCzechBirthNumber(candidate: string): boolean;
|
|
70
84
|
export declare function validateRomanianCnp(candidate: string): boolean;
|
|
71
85
|
export declare function validateHungarianTaj(candidate: string): boolean;
|
|
@@ -492,6 +492,13 @@ export function validatePolishPesel(candidate) {
|
|
|
492
492
|
total += d[i] * weights[i];
|
|
493
493
|
return d[10] === (10 - (total % 10)) % 10;
|
|
494
494
|
}
|
|
495
|
+
/** Polish domestic account number (NRB): the PL IBAN without its "PL", so the IBAN's mod-97 decides it (rules-engine#93). */
|
|
496
|
+
export function validatePolishNrb(candidate) {
|
|
497
|
+
const c = candidate.replace(/[\s\-]/g, "");
|
|
498
|
+
if (!/^\d{26}$/.test(c))
|
|
499
|
+
return false;
|
|
500
|
+
return validateIban("PL" + c);
|
|
501
|
+
}
|
|
495
502
|
export function validatePolishNip(candidate) {
|
|
496
503
|
const c = candidate.replace(/[\s\-]/g, "");
|
|
497
504
|
if (c.length !== 10 || !/^\d{10}$/.test(c))
|
|
@@ -503,6 +510,43 @@ export function validatePolishNip(candidate) {
|
|
|
503
510
|
total += d[i] * weights[i];
|
|
504
511
|
return total % 11 === d[9];
|
|
505
512
|
}
|
|
513
|
+
/**
|
|
514
|
+
* Polish identity card (dowód osobisty): 3 letters + 6 digits, the first digit
|
|
515
|
+
* a check digit. Letters A=10 … Z=35, weights 7,3,1,9,7,3,1,7,3: the weighted
|
|
516
|
+
* sum of all nine is 0 mod 10 exactly when the check digit is right (rules-engine#75).
|
|
517
|
+
*/
|
|
518
|
+
export function validatePolishIdCard(candidate) {
|
|
519
|
+
const c = candidate.replace(/\s/g, "").toUpperCase();
|
|
520
|
+
if (!/^[A-Z]{3}\d{6}$/.test(c))
|
|
521
|
+
return false;
|
|
522
|
+
const weights = [7, 3, 1, 9, 7, 3, 1, 7, 3];
|
|
523
|
+
let total = 0;
|
|
524
|
+
for (let i = 0; i < 9; i++) {
|
|
525
|
+
const ch = c[i];
|
|
526
|
+
total += (/[A-Z]/.test(ch) ? ch.charCodeAt(0) - 55 : Number(ch)) * weights[i];
|
|
527
|
+
}
|
|
528
|
+
return total % 10 === 0;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Polish REGON: 9 digits, or 14 for a local unit, each with a mod-11 check (a
|
|
532
|
+
* remainder of 10 counts as 0). A 14-digit REGON begins with its parent's valid
|
|
533
|
+
* 9-digit REGON (rules-engine#76).
|
|
534
|
+
*/
|
|
535
|
+
export function validatePolishRegon(candidate) {
|
|
536
|
+
const c = candidate.replace(/[\s\-]/g, "");
|
|
537
|
+
if (!/^\d+$/.test(c) || (c.length !== 9 && c.length !== 14))
|
|
538
|
+
return false;
|
|
539
|
+
const check = (digits, weights) => {
|
|
540
|
+
let total = 0;
|
|
541
|
+
for (let i = 0; i < weights.length; i++)
|
|
542
|
+
total += Number(digits[i]) * weights[i];
|
|
543
|
+
const rest = total % 11;
|
|
544
|
+
return (rest === 10 ? 0 : rest) === Number(digits[weights.length]);
|
|
545
|
+
};
|
|
546
|
+
if (!check(c.slice(0, 9), [8, 9, 2, 3, 4, 5, 6, 7]))
|
|
547
|
+
return false;
|
|
548
|
+
return c.length === 9 || check(c, [2, 4, 8, 5, 0, 9, 7, 3, 6, 1, 2, 4, 8]);
|
|
549
|
+
}
|
|
506
550
|
// Days per month for the rodné číslo date check. February is 29 because the
|
|
507
551
|
// century is not recoverable from a two-digit year, so a leap year cannot be
|
|
508
552
|
// ruled out -- being permissive by one day is the right direction here.
|
|
@@ -753,6 +797,9 @@ export const VALIDATORS = {
|
|
|
753
797
|
portuguese_nif: validatePortugueseNif,
|
|
754
798
|
polish_pesel: validatePolishPesel,
|
|
755
799
|
polish_nip: validatePolishNip,
|
|
800
|
+
polish_nrb: validatePolishNrb,
|
|
801
|
+
polish_id_card: validatePolishIdCard,
|
|
802
|
+
polish_regon: validatePolishRegon,
|
|
756
803
|
czech_birth_number: validateCzechBirthNumber,
|
|
757
804
|
romanian_cnp: validateRomanianCnp,
|
|
758
805
|
hungarian_taj: validateHungarianTaj,
|
package/dist/esm/sdk.d.ts
CHANGED
|
@@ -22,11 +22,51 @@ type LabelFor = (det: Detection, slice: string) => string;
|
|
|
22
22
|
*/
|
|
23
23
|
export declare function applyReplacements(text: string, detections: Detection[], labelFor: LabelFor): string;
|
|
24
24
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
25
|
+
* One label in the text sent to the cloud tier: where it sits in the masked
|
|
26
|
+
* text, and the extent of the original it replaced —
|
|
27
|
+
* `[maskedStart, maskedEnd, start, end]`.
|
|
28
28
|
*/
|
|
29
|
-
export
|
|
29
|
+
export type WireLabel = [number, number, number, number];
|
|
30
|
+
/**
|
|
31
|
+
* Bracket-mask `text` and record where each label landed.
|
|
32
|
+
*
|
|
33
|
+
* Returns what is sent to the cloud tier, plus one `WireLabel` per label.
|
|
34
|
+
* Everything between labels is copied unchanged, so the labels are all it
|
|
35
|
+
* takes to carry an offset from the masked text back to the original.
|
|
36
|
+
*
|
|
37
|
+
* Always `[TYPE]`, whatever label scheme the caller asked for. That is the
|
|
38
|
+
* syntax the model was trained on, where a placeholder means "already handled,
|
|
39
|
+
* do not list it". A token or a numbered label would be read as ordinary text,
|
|
40
|
+
* so the format on the wire is a model contract and not an output option.
|
|
41
|
+
*/
|
|
42
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
43
|
+
export declare function maskForCloud(text: string, detections: Detection[]): [string, WireLabel[]];
|
|
44
|
+
/**
|
|
45
|
+
* Carry the service's spans from the masked text onto the original.
|
|
46
|
+
*
|
|
47
|
+
* The service only ever saw `masked`, so its offsets index that. An offset in
|
|
48
|
+
* copied text shifts by whatever the labels before it added or removed. An
|
|
49
|
+
* offset inside a label has no counterpart in the original, so it snaps
|
|
50
|
+
* outward to the edge of the value the label replaced: an address the model
|
|
51
|
+
* reports around `[POSTAL_CODE]` covers the postal code, never half of it.
|
|
52
|
+
* Over-masking is the safe direction.
|
|
53
|
+
*
|
|
54
|
+
* A span lying wholly inside one label describes the label, not the document.
|
|
55
|
+
* It is dropped; the local detection behind the label stands.
|
|
56
|
+
*
|
|
57
|
+
* Throws `Unplaceable` when a span does not match the text that was sent. Such
|
|
58
|
+
* a span cannot be placed, and masking where it points would cover the wrong
|
|
59
|
+
* characters and leave the right ones in the clear.
|
|
60
|
+
*/
|
|
61
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
62
|
+
export declare function ontoOriginal(spans: Detection[], masked: string, text: string, labels: WireLabel[], Unplaceable: new (message: string) => Error): Detection[];
|
|
63
|
+
/**
|
|
64
|
+
* Characters a token suffix is drawn from. No vowels (and no Y), so a suffix
|
|
65
|
+
* never spells a word; no 0/1/I/O, so it survives being read back by a person;
|
|
66
|
+
* no underscore, so the type prefix stays unambiguous. A, E, U and Y were in it
|
|
67
|
+
* until rules-engine#55, against this comment.
|
|
68
|
+
*/
|
|
69
|
+
export declare const TOKEN_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ23456789";
|
|
30
70
|
export declare const TOKEN_SUFFIX_LENGTH = 4;
|
|
31
71
|
/**
|
|
32
72
|
* Maps each distinct PII value in one call to a reversible token.
|
|
@@ -64,6 +104,15 @@ export declare class TokenMapper {
|
|
|
64
104
|
* restored, since leaving a token behind is the worse failure.
|
|
65
105
|
*/
|
|
66
106
|
export declare function restore(text: string, tokens: Record<string, string>): string;
|
|
107
|
+
/**
|
|
108
|
+
* The entity types a caller asked to leave in the text, by canonical name.
|
|
109
|
+
*
|
|
110
|
+
* Names are matched case-insensitively and legacy names resolve (`IBAN` is
|
|
111
|
+
* `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A custom pattern's name counts
|
|
112
|
+
* once it is registered. Anything else throws: a typo in a type meant to be
|
|
113
|
+
* switched off would otherwise switch nothing off (rules-engine#125).
|
|
114
|
+
*/
|
|
115
|
+
export declare function normalizeExcludeTypes(values: unknown, customNames?: ReadonlySet<string>, param?: string): Set<string>;
|
|
67
116
|
export interface RedactOptions {
|
|
68
117
|
/**
|
|
69
118
|
* Scope. Detections attributed elsewhere are flagged `outOfScope`, never
|
|
@@ -104,8 +153,24 @@ export interface RedactOptions {
|
|
|
104
153
|
* EMAIL and URL only, and covers subdomains. Merged with the instance list.
|
|
105
154
|
*/
|
|
106
155
|
allowlistDomains?: string[] | null;
|
|
156
|
+
/**
|
|
157
|
+
* Entity types to leave in the text: neither in `detections` nor masked, and
|
|
158
|
+
* each such detection reported in `RedactResult.exempted` (`ruleKind:
|
|
159
|
+
* "type"`). Names are case-insensitive and legacy names resolve; unknown
|
|
160
|
+
* names throw. Merged with the instance's list. Default: nothing excluded.
|
|
161
|
+
* In `mode: "cloud"` an excluded rules type is still masked in the text sent
|
|
162
|
+
* to the service — this decides what comes back, not what may leave.
|
|
163
|
+
*/
|
|
164
|
+
excludeTypes?: string[] | null;
|
|
107
165
|
detectDates?: boolean;
|
|
108
166
|
cache?: boolean;
|
|
167
|
+
/**
|
|
168
|
+
* `mode: "cloud"` only. Sent as the request's `Idempotency-Key`: the same key
|
|
169
|
+
* returns the same job instead of running (and billing) the document again,
|
|
170
|
+
* which is what a caller retrying after its own timeout needs. One is
|
|
171
|
+
* generated per call when omitted.
|
|
172
|
+
*/
|
|
173
|
+
idempotencyKey?: string;
|
|
109
174
|
}
|
|
110
175
|
export declare class EuRedact {
|
|
111
176
|
private engine;
|
|
@@ -114,16 +179,22 @@ export declare class EuRedact {
|
|
|
114
179
|
private maxInputLength;
|
|
115
180
|
private allowlist;
|
|
116
181
|
private allowlistDomains;
|
|
182
|
+
private customNames;
|
|
183
|
+
private excludeTypes;
|
|
117
184
|
/**
|
|
118
185
|
* @param options.maxInputLength Longest document `redact` accepts, in characters.
|
|
119
186
|
* @param options.allowlist Values never to redact, for every call on this
|
|
120
187
|
* instance — an organisation's own name, its own addresses. Merged with
|
|
121
188
|
* the per-call `allowlist`. Matched whole, case-insensitively.
|
|
189
|
+
* @param options.excludeTypes Entity types never reported or masked, for
|
|
190
|
+
* every call on this instance, e.g. `["JOB_TITLE", "FINANCIAL_AMOUNT"]`.
|
|
191
|
+
* Merged with the per-call value. Unknown names throw.
|
|
122
192
|
*/
|
|
123
193
|
constructor(options?: {
|
|
124
194
|
maxInputLength?: number;
|
|
125
195
|
allowlist?: string[] | null;
|
|
126
196
|
allowlistDomains?: string[] | null;
|
|
197
|
+
excludeTypes?: string[] | null;
|
|
127
198
|
});
|
|
128
199
|
addCustomPattern(name: string, pattern: string): void;
|
|
129
200
|
/** Clear the result cache and referential integrity mappings.
|
|
@@ -131,6 +202,8 @@ export declare class EuRedact {
|
|
|
131
202
|
clear(): void;
|
|
132
203
|
/** The instance allowlist merged with a call's, in comparison form. */
|
|
133
204
|
private allowedFor;
|
|
205
|
+
/** The instance's excluded types merged with a call's. */
|
|
206
|
+
private excludedFor;
|
|
134
207
|
/** The instance domain list merged with a call's. */
|
|
135
208
|
private domainsFor;
|
|
136
209
|
/** The label function for one call, given its output options. */
|
package/dist/esm/sdk.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomIndex } from "./platform.js";
|
|
2
2
|
import { ResultCache } from "./cache.js";
|
|
3
3
|
import { normalize, mapOffsetToOriginal } from "./normalizer.js";
|
|
4
4
|
import { RuleEngine } from "./rules/engine.js";
|
|
5
5
|
import { weightsToRanking } from "./rules/evidence.js";
|
|
6
|
-
import { EntityType } from "./types.js";
|
|
6
|
+
import { EntityType, canonicalType } from "./types.js";
|
|
7
7
|
const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
|
|
8
8
|
/**
|
|
9
9
|
* Splice a label over every detection and return the masked text.
|
|
@@ -60,7 +60,8 @@ export function applyReplacements(text, detections, labelFor) {
|
|
|
60
60
|
* do not list it". A token or a numbered label would be read as ordinary text,
|
|
61
61
|
* so the format on the wire is a model contract and not an output option.
|
|
62
62
|
*/
|
|
63
|
-
|
|
63
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
64
|
+
export function maskForCloud(text, detections) {
|
|
64
65
|
const parts = [];
|
|
65
66
|
const labels = [];
|
|
66
67
|
let pos = 0;
|
|
@@ -110,7 +111,8 @@ function labelBefore(labels, offset, strict) {
|
|
|
110
111
|
* a span cannot be placed, and masking where it points would cover the wrong
|
|
111
112
|
* characters and leave the right ones in the clear.
|
|
112
113
|
*/
|
|
113
|
-
|
|
114
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
115
|
+
export function ontoOriginal(spans, masked, text, labels, Unplaceable) {
|
|
114
116
|
const placed = [];
|
|
115
117
|
for (const span of spans) {
|
|
116
118
|
if (!(0 <= span.start && span.start <= span.end && span.end <= masked.length) ||
|
|
@@ -139,11 +141,12 @@ function ontoOriginal(spans, masked, text, labels, Unplaceable) {
|
|
|
139
141
|
return placed;
|
|
140
142
|
}
|
|
141
143
|
/**
|
|
142
|
-
* Characters a token suffix is drawn from. No vowels, so a suffix
|
|
143
|
-
* a word; no 0/1/I/O, so it survives being read back by a person;
|
|
144
|
-
* underscore, so the type prefix stays unambiguous.
|
|
144
|
+
* Characters a token suffix is drawn from. No vowels (and no Y), so a suffix
|
|
145
|
+
* never spells a word; no 0/1/I/O, so it survives being read back by a person;
|
|
146
|
+
* no underscore, so the type prefix stays unambiguous. A, E, U and Y were in it
|
|
147
|
+
* until rules-engine#55, against this comment.
|
|
145
148
|
*/
|
|
146
|
-
export const TOKEN_ALPHABET = "
|
|
149
|
+
export const TOKEN_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ23456789";
|
|
147
150
|
export const TOKEN_SUFFIX_LENGTH = 4;
|
|
148
151
|
const TOKEN_MAX_DRAWS = 100;
|
|
149
152
|
function escapeRegExp(s) {
|
|
@@ -191,7 +194,7 @@ export class TokenMapper {
|
|
|
191
194
|
}
|
|
192
195
|
let suffix = "";
|
|
193
196
|
for (let i = 0; i < TOKEN_SUFFIX_LENGTH; i++) {
|
|
194
|
-
suffix += TOKEN_ALPHABET[
|
|
197
|
+
suffix += TOKEN_ALPHABET[randomIndex(TOKEN_ALPHABET.length)];
|
|
195
198
|
}
|
|
196
199
|
token = prefix + suffix;
|
|
197
200
|
} while (this.taken.has(token));
|
|
@@ -350,6 +353,70 @@ function applyAllowlist(text, detections, allowed, domains) {
|
|
|
350
353
|
}
|
|
351
354
|
return [kept, exempted];
|
|
352
355
|
}
|
|
356
|
+
const KNOWN_TYPES = new Set(Object.values(EntityType));
|
|
357
|
+
/**
|
|
358
|
+
* The entity types a caller asked to leave in the text, by canonical name.
|
|
359
|
+
*
|
|
360
|
+
* Names are matched case-insensitively and legacy names resolve (`IBAN` is
|
|
361
|
+
* `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A custom pattern's name counts
|
|
362
|
+
* once it is registered. Anything else throws: a typo in a type meant to be
|
|
363
|
+
* switched off would otherwise switch nothing off (rules-engine#125).
|
|
364
|
+
*/
|
|
365
|
+
export function normalizeExcludeTypes(values, customNames = new Set(), param = "excludeTypes") {
|
|
366
|
+
const out = new Set();
|
|
367
|
+
if (values === null || values === undefined)
|
|
368
|
+
return out;
|
|
369
|
+
checkAllowlistArg(values, param);
|
|
370
|
+
for (const v of values) {
|
|
371
|
+
if (typeof v !== "string") {
|
|
372
|
+
throw new TypeError(`${param} entries must be type names, not ${typeof v}`);
|
|
373
|
+
}
|
|
374
|
+
if (customNames.has(v)) {
|
|
375
|
+
out.add(v);
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
const name = canonicalType(v);
|
|
379
|
+
if (!KNOWN_TYPES.has(name)) {
|
|
380
|
+
throw new Error(`unknown entity type ${JSON.stringify(v)} in ${param}. Known types: ` +
|
|
381
|
+
[...KNOWN_TYPES].sort().join(", ") +
|
|
382
|
+
(customNames.size ? `; custom patterns: ${[...customNames].sort().join(", ")}` : ""));
|
|
383
|
+
}
|
|
384
|
+
out.add(name);
|
|
385
|
+
}
|
|
386
|
+
return out;
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Split off the detections of the types the caller excluded. Applied to each
|
|
390
|
+
* span's final type, after ranking: dropping an excluded type's candidates
|
|
391
|
+
* earlier would let the next-best type claim the same span.
|
|
392
|
+
*/
|
|
393
|
+
function applyTypeExclusion(text, detections, excluded) {
|
|
394
|
+
if (excluded.size === 0)
|
|
395
|
+
return [detections, []];
|
|
396
|
+
const kept = [];
|
|
397
|
+
const exempted = [];
|
|
398
|
+
for (const d of detections) {
|
|
399
|
+
const name = String(d.entityType);
|
|
400
|
+
if (excluded.has(name)) {
|
|
401
|
+
exempted.push({
|
|
402
|
+
entityType: d.entityType, start: d.start, end: d.end,
|
|
403
|
+
text: text.slice(d.start, d.end), rule: name, ruleKind: "type",
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
else
|
|
407
|
+
kept.push(d);
|
|
408
|
+
}
|
|
409
|
+
return [kept, exempted];
|
|
410
|
+
}
|
|
411
|
+
/** Allowlist first, then excluded types; every exemption in text order. */
|
|
412
|
+
function exempt(text, detections, allowed, domains, excluded) {
|
|
413
|
+
const [afterValues, byValue] = applyAllowlist(text, detections, allowed, domains);
|
|
414
|
+
const [kept, byType] = applyTypeExclusion(text, afterValues, excluded);
|
|
415
|
+
const exempted = [...byValue, ...byType];
|
|
416
|
+
if (byValue.length && byType.length)
|
|
417
|
+
exempted.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
418
|
+
return [kept, exempted];
|
|
419
|
+
}
|
|
353
420
|
/** Two label schemes for the same spans cannot both apply. */
|
|
354
421
|
function checkLabelOptions(options) {
|
|
355
422
|
if (options.tokenize && options.referentialIntegrity) {
|
|
@@ -430,19 +497,26 @@ export class EuRedact {
|
|
|
430
497
|
maxInputLength;
|
|
431
498
|
allowlist;
|
|
432
499
|
allowlistDomains;
|
|
500
|
+
customNames = new Set();
|
|
501
|
+
excludeTypes;
|
|
433
502
|
/**
|
|
434
503
|
* @param options.maxInputLength Longest document `redact` accepts, in characters.
|
|
435
504
|
* @param options.allowlist Values never to redact, for every call on this
|
|
436
505
|
* instance — an organisation's own name, its own addresses. Merged with
|
|
437
506
|
* the per-call `allowlist`. Matched whole, case-insensitively.
|
|
507
|
+
* @param options.excludeTypes Entity types never reported or masked, for
|
|
508
|
+
* every call on this instance, e.g. `["JOB_TITLE", "FINANCIAL_AMOUNT"]`.
|
|
509
|
+
* Merged with the per-call value. Unknown names throw.
|
|
438
510
|
*/
|
|
439
511
|
constructor(options) {
|
|
440
512
|
this.maxInputLength = options?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
|
|
441
513
|
this.allowlist = normalizeAllowlist(options?.allowlist);
|
|
442
514
|
this.allowlistDomains = normalizeDomains(options?.allowlistDomains);
|
|
515
|
+
this.excludeTypes = normalizeExcludeTypes(options?.excludeTypes);
|
|
443
516
|
}
|
|
444
517
|
addCustomPattern(name, pattern) {
|
|
445
518
|
this.engine.addCustomPattern(name, pattern);
|
|
519
|
+
this.customNames.add(name);
|
|
446
520
|
this.cache.clear();
|
|
447
521
|
}
|
|
448
522
|
/** Clear the result cache and referential integrity mappings.
|
|
@@ -458,6 +532,13 @@ export class EuRedact {
|
|
|
458
532
|
allowed.set(k, v);
|
|
459
533
|
return allowed;
|
|
460
534
|
}
|
|
535
|
+
/** The instance's excluded types merged with a call's. */
|
|
536
|
+
excludedFor(options) {
|
|
537
|
+
const excluded = new Set(this.excludeTypes);
|
|
538
|
+
for (const t of normalizeExcludeTypes(options.excludeTypes, this.customNames))
|
|
539
|
+
excluded.add(t);
|
|
540
|
+
return excluded;
|
|
541
|
+
}
|
|
461
542
|
/** The instance domain list merged with a call's. */
|
|
462
543
|
domainsFor(options) {
|
|
463
544
|
const domains = new Map(this.allowlistDomains);
|
|
@@ -514,22 +595,26 @@ export class EuRedact {
|
|
|
514
595
|
}
|
|
515
596
|
const allowed = this.allowedFor(options);
|
|
516
597
|
const domains = this.domainsFor(options);
|
|
598
|
+
const excluded = this.excludedFor(options);
|
|
517
599
|
const client = new CloudClient();
|
|
518
600
|
// Dates are always on, whatever detectDates says: that is the rules output
|
|
519
601
|
// the model was trained against, and a date of birth the rules can place
|
|
520
602
|
// has no reason to travel. It is the one ignored option that is safe to
|
|
521
603
|
// ignore — it can only cause MORE to be detected, never less.
|
|
522
604
|
//
|
|
523
|
-
// No allowlist and no tokens on this pass. An exemption
|
|
524
|
-
// caller wants back, not what may leave
|
|
605
|
+
// No allowlist, no excluded types and no tokens on this pass. An exemption
|
|
606
|
+
// says what the caller wants back, not what may leave: an excluded EMAIL is
|
|
607
|
+
// still masked in the request. All three are applied below, after the
|
|
525
608
|
// response, to the merged spans.
|
|
526
609
|
const local = this.redactRules(text, { countries, detectDates: true, cache: options.cache }, new Map(), new Map());
|
|
527
610
|
const [masked, labels] = maskForCloud(text, local.detections);
|
|
528
611
|
// The client converts the service's code-point offsets to UTF-16 units of
|
|
529
612
|
// the text it sent, so everything from here on is in one unit.
|
|
530
|
-
const remote = await client.redact(masked, {
|
|
613
|
+
const remote = await client.redact(masked, {
|
|
614
|
+
country: countries[0], idempotencyKey: options.idempotencyKey,
|
|
615
|
+
});
|
|
531
616
|
// A new array throughout: `local` may be the cached rules result.
|
|
532
|
-
const [detections, exempted] =
|
|
617
|
+
const [detections, exempted] = exempt(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains, excluded);
|
|
533
618
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
534
619
|
const tokenMapper = options.tokenize ? new TokenMapper(text, detections) : null;
|
|
535
620
|
return {
|
|
@@ -539,6 +624,7 @@ export class EuRedact {
|
|
|
539
624
|
source: "cloud",
|
|
540
625
|
tokens: tokenMapper ? tokenMapper.tokens : {},
|
|
541
626
|
exempted,
|
|
627
|
+
...(remote.cloud ? { cloud: remote.cloud } : {}),
|
|
542
628
|
};
|
|
543
629
|
}
|
|
544
630
|
redact(text, options = {}) {
|
|
@@ -547,6 +633,7 @@ export class EuRedact {
|
|
|
547
633
|
checkLabelOptions(options);
|
|
548
634
|
const allowed = this.allowedFor(options);
|
|
549
635
|
const domains = this.domainsFor(options);
|
|
636
|
+
const excluded = this.excludedFor(options);
|
|
550
637
|
const requestedMode = options.mode ?? "rules";
|
|
551
638
|
if (requestedMode === "cloud") {
|
|
552
639
|
// Cannot be served synchronously: the cloud tier is a network call, and
|
|
@@ -560,10 +647,13 @@ export class EuRedact {
|
|
|
560
647
|
if (requestedMode !== "rules") {
|
|
561
648
|
throw new Error(`unknown mode ${JSON.stringify(requestedMode)}: expected "rules" or "cloud"`);
|
|
562
649
|
}
|
|
563
|
-
|
|
650
|
+
if (options.idempotencyKey !== undefined) {
|
|
651
|
+
throw new Error('idempotencyKey applies to mode: "cloud" only: a rules-only redaction makes no request to deduplicate');
|
|
652
|
+
}
|
|
653
|
+
return this.redactRules(text, options, allowed, domains, excluded);
|
|
564
654
|
}
|
|
565
655
|
/** The local pipeline: normalise, detect, exempt, mask. */
|
|
566
|
-
redactRules(text, options, allowed, domains) {
|
|
656
|
+
redactRules(text, options, allowed, domains, excluded = new Set()) {
|
|
567
657
|
if (text.length > this.maxInputLength) {
|
|
568
658
|
throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
|
|
569
659
|
`(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
|
|
@@ -586,7 +676,8 @@ export class EuRedact {
|
|
|
586
676
|
// JSON rather than a joined string: an entry may itself contain the
|
|
587
677
|
// separator, and two different lists must never share a key.
|
|
588
678
|
const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
|
|
589
|
-
+ (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "")
|
|
679
|
+
+ (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "")
|
|
680
|
+
+ (excluded.size ? "|excl=" + JSON.stringify([...excluded].sort()) : "");
|
|
590
681
|
const cacheMode = `rules|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
|
|
591
682
|
let cacheKey;
|
|
592
683
|
if (cache) {
|
|
@@ -610,7 +701,7 @@ export class EuRedact {
|
|
|
610
701
|
detections = detections.filter(d => !DATE_TYPES.has(d.entityType));
|
|
611
702
|
}
|
|
612
703
|
let exempted;
|
|
613
|
-
[detections, exempted] =
|
|
704
|
+
[detections, exempted] = exempt(text, detections, allowed, domains, excluded);
|
|
614
705
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
615
706
|
const tokenMapper = tokenize ? new TokenMapper(text, detections) : null;
|
|
616
707
|
const redacted = applyReplacements(text, detections, this.labelFor(referentialIntegrity, tokenMapper));
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -69,7 +69,8 @@ export declare enum EntityType {
|
|
|
69
69
|
BIOMETRIC_REF = "BIOMETRIC_REF",
|
|
70
70
|
FINANCIAL_AMOUNT = "FINANCIAL_AMOUNT",
|
|
71
71
|
QUASI_IDENTIFIER = "QUASI_IDENTIFIER",
|
|
72
|
-
|
|
72
|
+
/** Legacy alias of SECRET: every credential, the model's included, is SECRET (rules-engine#111). */
|
|
73
|
+
CREDENTIAL = "SECRET",
|
|
73
74
|
URL = "URL",
|
|
74
75
|
OTHER = "OTHER"
|
|
75
76
|
}
|
|
@@ -116,7 +117,13 @@ export interface Detection {
|
|
|
116
117
|
text: string;
|
|
117
118
|
source: DetectionSource;
|
|
118
119
|
country: string | null;
|
|
119
|
-
|
|
120
|
+
/**
|
|
121
|
+
* How the type was arrived at: "high", "medium" or "low" for a rules
|
|
122
|
+
* detection. `null` for a span the cloud model found and reported no
|
|
123
|
+
* confidence for: the model emits none today, and "high" would claim the
|
|
124
|
+
* checksum-backed evidence a rules detection has (rules-engine#108).
|
|
125
|
+
*/
|
|
126
|
+
confidence: string | null;
|
|
120
127
|
/**
|
|
121
128
|
* How strongly the document supports {@link Detection.country}, in [0, 1].
|
|
122
129
|
*
|
|
@@ -134,7 +141,7 @@ export interface Detection {
|
|
|
134
141
|
outOfScope?: boolean;
|
|
135
142
|
}
|
|
136
143
|
/**
|
|
137
|
-
* A detection the allowlist kept out of the redacted output.
|
|
144
|
+
* A detection the allowlist or `excludeTypes` kept out of the redacted output.
|
|
138
145
|
*
|
|
139
146
|
* An exemption is a deliberate decision to leave a direct identifier in a
|
|
140
147
|
* document that otherwise claims to be redacted, so it is reported rather than
|
|
@@ -146,10 +153,58 @@ export interface Exemption {
|
|
|
146
153
|
start: number;
|
|
147
154
|
end: number;
|
|
148
155
|
text: string;
|
|
149
|
-
/** The allowlist entry that matched, as the caller wrote it
|
|
156
|
+
/** The allowlist entry that matched, as the caller wrote it; for an
|
|
157
|
+
* excluded type, the type's canonical name. */
|
|
150
158
|
rule: string;
|
|
151
|
-
/** `"value"` for the literal allowlist, `"domain"` for allowlistDomains
|
|
152
|
-
|
|
159
|
+
/** `"value"` for the literal allowlist, `"domain"` for allowlistDomains,
|
|
160
|
+
* `"type"` for excludeTypes (rules-engine#125). */
|
|
161
|
+
ruleKind: "value" | "domain" | "type";
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* One reason a cloud request cost what it did. `code` is one of the service's
|
|
165
|
+
* factor codes (`rules_only`, `document_length`, `instructions`,
|
|
166
|
+
* `dense_document`, `special_category_check`, `long_document`, `batch_rate`),
|
|
167
|
+
* kept as a string so a code added later on the service still arrives.
|
|
168
|
+
*/
|
|
169
|
+
export interface UsageFactor {
|
|
170
|
+
code: string;
|
|
171
|
+
/** One sentence explaining the factor, written by the service. */
|
|
172
|
+
detail: string;
|
|
173
|
+
/** `special_category_check` only: the kinds of data that triggered it. */
|
|
174
|
+
types?: string[];
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* What a cloud request cost, and why (rules-engine#89). A total and its
|
|
178
|
+
* reasons, never a cost per step; `credits` is `round(tokens * billingRate)`,
|
|
179
|
+
* the formula the service debits by.
|
|
180
|
+
*/
|
|
181
|
+
export interface Usage {
|
|
182
|
+
tokens: number;
|
|
183
|
+
billingRate: number;
|
|
184
|
+
credits: number;
|
|
185
|
+
factors: UsageFactor[];
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Something the model found that could not be placed in the document: the
|
|
189
|
+
* service could not match it to a position in what it was sent, so nothing was
|
|
190
|
+
* masked for it. Reported so a caller can decide, rather than never being told.
|
|
191
|
+
*/
|
|
192
|
+
export interface Unlocated {
|
|
193
|
+
text: string;
|
|
194
|
+
entityType: EntityType | string;
|
|
195
|
+
}
|
|
196
|
+
/** What the cloud service said about a request, beside its spans (rules-engine#89). */
|
|
197
|
+
export interface CloudInfo {
|
|
198
|
+
/** The service's job id: for support, for `Jobs.retrieve`, and to recognise an
|
|
199
|
+
* idempotent replay. `null` for a batch document. */
|
|
200
|
+
jobId: string | null;
|
|
201
|
+
/** The public name of the model that answered, `null` when only rules ran. */
|
|
202
|
+
modelVersion: string | null;
|
|
203
|
+
/** What the request cost and why. Absent when the service does not report it
|
|
204
|
+
* (older versions, batch documents) or reported it malformed. */
|
|
205
|
+
usage?: Usage;
|
|
206
|
+
/** What the model found but could not place in the text it was sent. */
|
|
207
|
+
unlocated: Unlocated[];
|
|
153
208
|
}
|
|
154
209
|
export interface RedactResult {
|
|
155
210
|
redactedText: string;
|
|
@@ -188,6 +243,9 @@ export interface RedactResult {
|
|
|
188
243
|
/** Detections the allowlist kept out of the output, with the rule that
|
|
189
244
|
* matched. Empty unless an allowlist was in force. */
|
|
190
245
|
exempted: Exemption[];
|
|
246
|
+
/** What the cloud service reported beside the spans: job id, model, cost and
|
|
247
|
+
* anything it could not place. Absent on a rules-only result. */
|
|
248
|
+
cloud?: CloudInfo;
|
|
191
249
|
}
|
|
192
250
|
export interface PatternDef {
|
|
193
251
|
entityType: EntityType | string;
|
package/dist/esm/types.js
CHANGED
|
@@ -75,7 +75,8 @@ export var EntityType;
|
|
|
75
75
|
EntityType["BIOMETRIC_REF"] = "BIOMETRIC_REF";
|
|
76
76
|
EntityType["FINANCIAL_AMOUNT"] = "FINANCIAL_AMOUNT";
|
|
77
77
|
EntityType["QUASI_IDENTIFIER"] = "QUASI_IDENTIFIER";
|
|
78
|
-
|
|
78
|
+
/** Legacy alias of SECRET: every credential, the model's included, is SECRET (rules-engine#111). */
|
|
79
|
+
EntityType["CREDENTIAL"] = "SECRET";
|
|
79
80
|
EntityType["URL"] = "URL";
|
|
80
81
|
EntityType["OTHER"] = "OTHER";
|
|
81
82
|
})(EntityType || (EntityType = {}));
|
|
@@ -89,6 +90,8 @@ export const LEGACY_TYPE_ALIASES = {
|
|
|
89
90
|
NAME: "PERSON_NAME",
|
|
90
91
|
STREET_ADDRESS: "ADDRESS",
|
|
91
92
|
NATIONALITY_ETHNICITY: "SENSITIVE_ATTRIBUTE",
|
|
93
|
+
// The model labels prose credentials CREDENTIAL; the SDK reports every credential as SECRET (rules-engine#111).
|
|
94
|
+
CREDENTIAL: "SECRET",
|
|
92
95
|
};
|
|
93
96
|
/**
|
|
94
97
|
* Resolve a possibly-legacy type name to its canonical form.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
5
|
"homepage": "https://euredact.dev",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -20,10 +20,15 @@
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
},
|
|
23
|
+
"browser": {
|
|
24
|
+
"./dist/esm/platform.js": "./dist/esm/platform.browser.js",
|
|
25
|
+
"./dist/cjs/platform.js": "./dist/cjs/platform.browser.js"
|
|
26
|
+
},
|
|
23
27
|
"files": [
|
|
24
28
|
"dist",
|
|
25
29
|
"README.md",
|
|
26
|
-
"LICENSE"
|
|
30
|
+
"LICENSE",
|
|
31
|
+
"NOTICE"
|
|
27
32
|
],
|
|
28
33
|
"engines": {
|
|
29
34
|
"node": ">=16.0.0"
|
|
@@ -54,7 +59,7 @@
|
|
|
54
59
|
"build:cjs": "tsc -p tsconfig.cjs.json",
|
|
55
60
|
"postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
|
|
56
61
|
"build": "rm -rf dist && npm run build:esm && npm run build:cjs",
|
|
57
|
-
"test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:tokenize && npm run test:allowlist && npm run test:eval-scoring && npm run test:conformance",
|
|
62
|
+
"test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:cloud-api && npm run test:tokenize && npm run test:allowlist && npm run test:exclude-types && npm run test:eval-scoring && npm run test:platform && npm run test:batches && npm run test:conformance",
|
|
58
63
|
"benchmark": "node --import tsx src/__tests__/benchmark.ts",
|
|
59
64
|
"prepublishOnly": "npm run build",
|
|
60
65
|
"test:spec": "node --import tsx src/__tests__/spec.ts",
|
|
@@ -64,9 +69,13 @@
|
|
|
64
69
|
"test:inference": "node --import tsx src/__tests__/inference.ts",
|
|
65
70
|
"dump": "node --import tsx src/__tests__/dump.ts",
|
|
66
71
|
"test:cloud": "node --import tsx src/__tests__/cloud.ts",
|
|
72
|
+
"test:cloud-api": "node --import tsx src/__tests__/cloudApi.ts",
|
|
67
73
|
"test:tokenize": "node --import tsx src/__tests__/tokenize.ts",
|
|
68
74
|
"test:allowlist": "node --import tsx src/__tests__/allowlist.ts",
|
|
69
|
-
"test:
|
|
75
|
+
"test:exclude-types": "node --import tsx src/__tests__/excludeTypes.ts",
|
|
76
|
+
"test:eval-scoring": "node --import tsx src/__tests__/evalScoring.ts",
|
|
77
|
+
"test:platform": "node --import tsx src/__tests__/platform.ts",
|
|
78
|
+
"test:batches": "node --import tsx src/__tests__/batches.ts"
|
|
70
79
|
},
|
|
71
80
|
"devDependencies": {
|
|
72
81
|
"@types/node": "22.10.2",
|