euredact 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +658 -45
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +131 -18
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +138 -48
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +418 -23
- package/dist/cjs/rules/validators.js +103 -0
- package/dist/cjs/sdk.js +150 -44
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +90 -5
- package/dist/esm/cloud/client.js +126 -26
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +138 -48
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +417 -22
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +100 -0
- package/dist/esm/sdk.d.ts +63 -19
- package/dist/esm/sdk.js +148 -44
- package/dist/esm/types.d.ts +58 -2
- package/dist/esm/types.js +4 -1
- package/package.json +12 -4
|
@@ -32,7 +32,10 @@ exports.validateSpanishDni = validateSpanishDni;
|
|
|
32
32
|
exports.validateSpanishNie = validateSpanishNie;
|
|
33
33
|
exports.validatePortugueseNif = validatePortugueseNif;
|
|
34
34
|
exports.validatePolishPesel = validatePolishPesel;
|
|
35
|
+
exports.validatePolishNrb = validatePolishNrb;
|
|
35
36
|
exports.validatePolishNip = validatePolishNip;
|
|
37
|
+
exports.validatePolishIdCard = validatePolishIdCard;
|
|
38
|
+
exports.validatePolishRegon = validatePolishRegon;
|
|
36
39
|
exports.validateCzechBirthNumber = validateCzechBirthNumber;
|
|
37
40
|
exports.validateRomanianCnp = validateRomanianCnp;
|
|
38
41
|
exports.validateHungarianTaj = validateHungarianTaj;
|
|
@@ -513,10 +516,24 @@ function validatePortugueseNif(candidate) {
|
|
|
513
516
|
const check = remainder < 2 ? 0 : 11 - remainder;
|
|
514
517
|
return parseInt(c[8]) === check;
|
|
515
518
|
}
|
|
519
|
+
// PESEL carries the century in the month field: +0 is the 1900s, +20 the 2000s,
|
|
520
|
+
// +40 the 2100s, +60 the 2200s, +80 the 1800s. Without a date check this
|
|
521
|
+
// accepted 80666603210 -- day 66 -- as NATIONAL_ID at high confidence
|
|
522
|
+
// (rules-engine#43).
|
|
516
523
|
function validatePolishPesel(candidate) {
|
|
517
524
|
const c = candidate.replace(/[\s\-]/g, "");
|
|
518
525
|
if (c.length !== 11 || !/^\d{11}$/.test(c))
|
|
519
526
|
return false;
|
|
527
|
+
const rawMonth = parseInt(c.slice(2, 4), 10);
|
|
528
|
+
let month = null;
|
|
529
|
+
for (const off of [0, 20, 40, 60, 80]) {
|
|
530
|
+
if (rawMonth - off >= 1 && rawMonth - off <= 12) {
|
|
531
|
+
month = rawMonth - off;
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
if (month === null || !isRealDay(month, parseInt(c.slice(4, 6), 10)))
|
|
536
|
+
return false;
|
|
520
537
|
const d = c.split("").map(Number);
|
|
521
538
|
const weights = [1, 3, 7, 9, 1, 3, 7, 9, 1, 3];
|
|
522
539
|
let total = 0;
|
|
@@ -524,6 +541,13 @@ function validatePolishPesel(candidate) {
|
|
|
524
541
|
total += d[i] * weights[i];
|
|
525
542
|
return d[10] === (10 - (total % 10)) % 10;
|
|
526
543
|
}
|
|
544
|
+
/** Polish domestic account number (NRB): the PL IBAN without its "PL", so the IBAN's mod-97 decides it (rules-engine#93). */
|
|
545
|
+
function validatePolishNrb(candidate) {
|
|
546
|
+
const c = candidate.replace(/[\s\-]/g, "");
|
|
547
|
+
if (!/^\d{26}$/.test(c))
|
|
548
|
+
return false;
|
|
549
|
+
return validateIban("PL" + c);
|
|
550
|
+
}
|
|
527
551
|
function validatePolishNip(candidate) {
|
|
528
552
|
const c = candidate.replace(/[\s\-]/g, "");
|
|
529
553
|
if (c.length !== 10 || !/^\d{10}$/.test(c))
|
|
@@ -535,18 +559,90 @@ function validatePolishNip(candidate) {
|
|
|
535
559
|
total += d[i] * weights[i];
|
|
536
560
|
return total % 11 === d[9];
|
|
537
561
|
}
|
|
562
|
+
/**
|
|
563
|
+
* Polish identity card (dowód osobisty): 3 letters + 6 digits, the first digit
|
|
564
|
+
* a check digit. Letters A=10 … Z=35, weights 7,3,1,9,7,3,1,7,3: the weighted
|
|
565
|
+
* sum of all nine is 0 mod 10 exactly when the check digit is right (rules-engine#75).
|
|
566
|
+
*/
|
|
567
|
+
function validatePolishIdCard(candidate) {
|
|
568
|
+
const c = candidate.replace(/\s/g, "").toUpperCase();
|
|
569
|
+
if (!/^[A-Z]{3}\d{6}$/.test(c))
|
|
570
|
+
return false;
|
|
571
|
+
const weights = [7, 3, 1, 9, 7, 3, 1, 7, 3];
|
|
572
|
+
let total = 0;
|
|
573
|
+
for (let i = 0; i < 9; i++) {
|
|
574
|
+
const ch = c[i];
|
|
575
|
+
total += (/[A-Z]/.test(ch) ? ch.charCodeAt(0) - 55 : Number(ch)) * weights[i];
|
|
576
|
+
}
|
|
577
|
+
return total % 10 === 0;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Polish REGON: 9 digits, or 14 for a local unit, each with a mod-11 check (a
|
|
581
|
+
* remainder of 10 counts as 0). A 14-digit REGON begins with its parent's valid
|
|
582
|
+
* 9-digit REGON (rules-engine#76).
|
|
583
|
+
*/
|
|
584
|
+
function validatePolishRegon(candidate) {
|
|
585
|
+
const c = candidate.replace(/[\s\-]/g, "");
|
|
586
|
+
if (!/^\d+$/.test(c) || (c.length !== 9 && c.length !== 14))
|
|
587
|
+
return false;
|
|
588
|
+
const check = (digits, weights) => {
|
|
589
|
+
let total = 0;
|
|
590
|
+
for (let i = 0; i < weights.length; i++)
|
|
591
|
+
total += Number(digits[i]) * weights[i];
|
|
592
|
+
const rest = total % 11;
|
|
593
|
+
return (rest === 10 ? 0 : rest) === Number(digits[weights.length]);
|
|
594
|
+
};
|
|
595
|
+
if (!check(c.slice(0, 9), [8, 9, 2, 3, 4, 5, 6, 7]))
|
|
596
|
+
return false;
|
|
597
|
+
return c.length === 9 || check(c, [2, 4, 8, 5, 0, 9, 7, 3, 6, 1, 2, 4, 8]);
|
|
598
|
+
}
|
|
599
|
+
// Days per month for the rodné číslo date check. February is 29 because the
|
|
600
|
+
// century is not recoverable from a two-digit year, so a leap year cannot be
|
|
601
|
+
// ruled out -- being permissive by one day is the right direction here.
|
|
602
|
+
const RC_DAYS = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
603
|
+
// The calendar month a rodné číslo month field encodes, or null. Four ranges
|
|
604
|
+
// are legal: 1-12 is a man, +50 a woman, and since 2004 +20 marks a birth on a
|
|
605
|
+
// day whose sequence numbers were exhausted (so +70 is a woman on such a day).
|
|
606
|
+
// Whether `day` exists in `month`, February permissive at 29 because a
|
|
607
|
+
// two-digit year does not reveal the century.
|
|
608
|
+
function isRealDay(month, day) {
|
|
609
|
+
return month >= 1 && month <= 12 && day >= 1 && day <= RC_DAYS[month - 1];
|
|
610
|
+
}
|
|
611
|
+
function czechMonth(raw) {
|
|
612
|
+
for (const offset of [0, 20, 50, 70]) {
|
|
613
|
+
if (raw - offset >= 1 && raw - offset <= 12)
|
|
614
|
+
return raw - offset;
|
|
615
|
+
}
|
|
616
|
+
return null;
|
|
617
|
+
}
|
|
618
|
+
// The date is checked, not only the checksum. Without it the validator accepted
|
|
619
|
+
// an impossible month, and a Czech mobile number is the same shape -- nine
|
|
620
|
+
// digits opening 6 or 7 -- so any mobile that happened to pass mod 11 was typed
|
|
621
|
+
// NATIONAL_ID at high confidence: 606666032 (MM=66), 778836400 (MM=88),
|
|
622
|
+
// 728990603 (MM=89). 284 per corpus pass, the largest single false-positive
|
|
623
|
+
// bucket in the evaluation (rules-engine#37).
|
|
538
624
|
function validateCzechBirthNumber(candidate) {
|
|
539
625
|
const c = candidate.replace(/[/\s]/g, "");
|
|
540
626
|
if (!/^\d+$/.test(c) || (c.length !== 9 && c.length !== 10))
|
|
541
627
|
return false;
|
|
628
|
+
const month = czechMonth(parseInt(c.slice(2, 4), 10));
|
|
629
|
+
if (month === null)
|
|
630
|
+
return false;
|
|
631
|
+
const day = parseInt(c.slice(4, 6), 10);
|
|
632
|
+
if (day < 1 || day > RC_DAYS[month - 1])
|
|
633
|
+
return false;
|
|
542
634
|
if (c.length === 10)
|
|
543
635
|
return parseInt(c) % 11 === 0;
|
|
544
636
|
return true;
|
|
545
637
|
}
|
|
638
|
+
// S YY MM DD JJ NNN C -- the leading S carries the century, so the month needs
|
|
639
|
+
// no offset convention here (rules-engine#43).
|
|
546
640
|
function validateRomanianCnp(candidate) {
|
|
547
641
|
const c = candidate.replace(/[\s\-]/g, "");
|
|
548
642
|
if (c.length !== 13 || !/^\d{13}$/.test(c) || !"12345678".includes(c[0]))
|
|
549
643
|
return false;
|
|
644
|
+
if (!isRealDay(parseInt(c.slice(3, 5), 10), parseInt(c.slice(5, 7), 10)))
|
|
645
|
+
return false;
|
|
550
646
|
const key = "279146358279";
|
|
551
647
|
let total = 0;
|
|
552
648
|
for (let i = 0; i < 12; i++)
|
|
@@ -593,10 +689,14 @@ function validateCroatianOib(candidate) {
|
|
|
593
689
|
const check = (11 - product) % 10;
|
|
594
690
|
return check === parseInt(c[10]);
|
|
595
691
|
}
|
|
692
|
+
// DD MM YYY RR BBB K -- note EMSO opens with the *day*, where the Czech, Polish
|
|
693
|
+
// and Romanian forms open with the year or a century digit (rules-engine#43).
|
|
596
694
|
function validateSlovenianEmso(candidate) {
|
|
597
695
|
const c = candidate.replace(/[\s\-]/g, "");
|
|
598
696
|
if (c.length !== 13 || !/^\d{13}$/.test(c))
|
|
599
697
|
return false;
|
|
698
|
+
if (!isRealDay(parseInt(c.slice(2, 4), 10), parseInt(c.slice(0, 2), 10)))
|
|
699
|
+
return false;
|
|
600
700
|
const d = c.split("").map(Number);
|
|
601
701
|
const weights = [7, 6, 5, 4, 3, 2, 7, 6, 5, 4, 3, 2];
|
|
602
702
|
let total = 0;
|
|
@@ -746,6 +846,9 @@ exports.VALIDATORS = {
|
|
|
746
846
|
portuguese_nif: validatePortugueseNif,
|
|
747
847
|
polish_pesel: validatePolishPesel,
|
|
748
848
|
polish_nip: validatePolishNip,
|
|
849
|
+
polish_nrb: validatePolishNrb,
|
|
850
|
+
polish_id_card: validatePolishIdCard,
|
|
851
|
+
polish_regon: validatePolishRegon,
|
|
749
852
|
czech_birth_number: validateCzechBirthNumber,
|
|
750
853
|
romanian_cnp: validateRomanianCnp,
|
|
751
854
|
hungarian_taj: validateHungarianTaj,
|
package/dist/cjs/sdk.js
CHANGED
|
@@ -35,8 +35,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.EuRedact = exports.TokenMapper = exports.TOKEN_SUFFIX_LENGTH = exports.TOKEN_ALPHABET = void 0;
|
|
37
37
|
exports.applyReplacements = applyReplacements;
|
|
38
|
+
exports.maskForCloud = maskForCloud;
|
|
39
|
+
exports.ontoOriginal = ontoOriginal;
|
|
38
40
|
exports.restore = restore;
|
|
39
|
-
const
|
|
41
|
+
const platform_js_1 = require("./platform.js");
|
|
40
42
|
const cache_js_1 = require("./cache.js");
|
|
41
43
|
const normalizer_js_1 = require("./normalizer.js");
|
|
42
44
|
const engine_js_1 = require("./rules/engine.js");
|
|
@@ -87,11 +89,104 @@ function applyReplacements(text, detections, labelFor) {
|
|
|
87
89
|
return parts.join("");
|
|
88
90
|
}
|
|
89
91
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
92
|
+
* Bracket-mask `text` and record where each label landed.
|
|
93
|
+
*
|
|
94
|
+
* Returns what is sent to the cloud tier, plus one `WireLabel` per label.
|
|
95
|
+
* Everything between labels is copied unchanged, so the labels are all it
|
|
96
|
+
* takes to carry an offset from the masked text back to the original.
|
|
97
|
+
*
|
|
98
|
+
* Always `[TYPE]`, whatever label scheme the caller asked for. That is the
|
|
99
|
+
* syntax the model was trained on, where a placeholder means "already handled,
|
|
100
|
+
* do not list it". A token or a numbered label would be read as ordinary text,
|
|
101
|
+
* so the format on the wire is a model contract and not an output option.
|
|
102
|
+
*/
|
|
103
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
104
|
+
function maskForCloud(text, detections) {
|
|
105
|
+
const parts = [];
|
|
106
|
+
const labels = [];
|
|
107
|
+
let pos = 0;
|
|
108
|
+
let out = 0;
|
|
109
|
+
for (const det of detections) {
|
|
110
|
+
const start = Math.max(det.start, pos);
|
|
111
|
+
if (det.end <= start)
|
|
112
|
+
continue;
|
|
113
|
+
const label = `[${det.entityType}]`;
|
|
114
|
+
out += start - pos;
|
|
115
|
+
parts.push(text.slice(pos, start), label);
|
|
116
|
+
labels.push([out, out + label.length, start, det.end]);
|
|
117
|
+
out += label.length;
|
|
118
|
+
pos = det.end;
|
|
119
|
+
}
|
|
120
|
+
parts.push(text.slice(pos));
|
|
121
|
+
return [parts.join(""), labels];
|
|
122
|
+
}
|
|
123
|
+
/** Index of the last label starting at or before `offset` — strictly before
|
|
124
|
+
* when `strict` — or -1. */
|
|
125
|
+
function labelBefore(labels, offset, strict) {
|
|
126
|
+
let lo = 0;
|
|
127
|
+
let hi = labels.length;
|
|
128
|
+
while (lo < hi) {
|
|
129
|
+
const mid = (lo + hi) >>> 1;
|
|
130
|
+
if (strict ? labels[mid][0] < offset : labels[mid][0] <= offset)
|
|
131
|
+
lo = mid + 1;
|
|
132
|
+
else
|
|
133
|
+
hi = mid;
|
|
134
|
+
}
|
|
135
|
+
return lo - 1;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Carry the service's spans from the masked text onto the original.
|
|
139
|
+
*
|
|
140
|
+
* The service only ever saw `masked`, so its offsets index that. An offset in
|
|
141
|
+
* copied text shifts by whatever the labels before it added or removed. An
|
|
142
|
+
* offset inside a label has no counterpart in the original, so it snaps
|
|
143
|
+
* outward to the edge of the value the label replaced: an address the model
|
|
144
|
+
* reports around `[POSTAL_CODE]` covers the postal code, never half of it.
|
|
145
|
+
* Over-masking is the safe direction.
|
|
146
|
+
*
|
|
147
|
+
* A span lying wholly inside one label describes the label, not the document.
|
|
148
|
+
* It is dropped; the local detection behind the label stands.
|
|
149
|
+
*
|
|
150
|
+
* Throws `Unplaceable` when a span does not match the text that was sent. Such
|
|
151
|
+
* a span cannot be placed, and masking where it points would cover the wrong
|
|
152
|
+
* characters and leave the right ones in the clear.
|
|
153
|
+
*/
|
|
154
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
155
|
+
function ontoOriginal(spans, masked, text, labels, Unplaceable) {
|
|
156
|
+
const placed = [];
|
|
157
|
+
for (const span of spans) {
|
|
158
|
+
if (!(0 <= span.start && span.start <= span.end && span.end <= masked.length) ||
|
|
159
|
+
masked.slice(span.start, span.end) !== span.text) {
|
|
160
|
+
throw new Unplaceable("span offsets do not match the text that was sent; cannot place the " +
|
|
161
|
+
"service's detections in the document");
|
|
162
|
+
}
|
|
163
|
+
if (span.start === span.end)
|
|
164
|
+
continue;
|
|
165
|
+
let start = span.start;
|
|
166
|
+
let at = labelBefore(labels, span.start, false);
|
|
167
|
+
if (at >= 0) {
|
|
168
|
+
const [, maskedEnd, origStart, origEnd] = labels[at];
|
|
169
|
+
if (span.end <= maskedEnd)
|
|
170
|
+
continue;
|
|
171
|
+
start = span.start < maskedEnd ? origStart : origEnd + span.start - maskedEnd;
|
|
172
|
+
}
|
|
173
|
+
let end = span.end;
|
|
174
|
+
at = labelBefore(labels, span.end, true);
|
|
175
|
+
if (at >= 0) {
|
|
176
|
+
const [, maskedEnd, , origEnd] = labels[at];
|
|
177
|
+
end = origEnd + Math.max(0, span.end - maskedEnd);
|
|
178
|
+
}
|
|
179
|
+
placed.push({ ...span, start, end, text: text.slice(start, end) });
|
|
180
|
+
}
|
|
181
|
+
return placed;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Characters a token suffix is drawn from. No vowels (and no Y), so a suffix
|
|
185
|
+
* never spells a word; no 0/1/I/O, so it survives being read back by a person;
|
|
186
|
+
* no underscore, so the type prefix stays unambiguous. A, E, U and Y were in it
|
|
187
|
+
* until rules-engine#55, against this comment.
|
|
93
188
|
*/
|
|
94
|
-
exports.TOKEN_ALPHABET = "
|
|
189
|
+
exports.TOKEN_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ23456789";
|
|
95
190
|
exports.TOKEN_SUFFIX_LENGTH = 4;
|
|
96
191
|
const TOKEN_MAX_DRAWS = 100;
|
|
97
192
|
function escapeRegExp(s) {
|
|
@@ -139,7 +234,7 @@ class TokenMapper {
|
|
|
139
234
|
}
|
|
140
235
|
let suffix = "";
|
|
141
236
|
for (let i = 0; i < exports.TOKEN_SUFFIX_LENGTH; i++) {
|
|
142
|
-
suffix += exports.TOKEN_ALPHABET[(0,
|
|
237
|
+
suffix += exports.TOKEN_ALPHABET[(0, platform_js_1.randomIndex)(exports.TOKEN_ALPHABET.length)];
|
|
143
238
|
}
|
|
144
239
|
token = prefix + suffix;
|
|
145
240
|
} while (this.taken.has(token));
|
|
@@ -425,7 +520,16 @@ class EuRedact {
|
|
|
425
520
|
return (det) => `[${det.entityType}]`;
|
|
426
521
|
}
|
|
427
522
|
/**
|
|
428
|
-
*
|
|
523
|
+
* Run the rules here, then send what they left to the cloud tier.
|
|
524
|
+
*
|
|
525
|
+
* Local-first: the rules engine masks everything it can find on the caller's
|
|
526
|
+
* machine, and only that masked text is sent. The service looks for what has
|
|
527
|
+
* no shape to match on — names, employers, diagnoses — and answers with
|
|
528
|
+
* spans relative to the text it received, which are mapped back onto the
|
|
529
|
+
* original here.
|
|
530
|
+
*
|
|
531
|
+
* This is minimisation, not an exemption: what the rules miss, and the names
|
|
532
|
+
* and diagnoses the model is there to find, still travel.
|
|
429
533
|
*
|
|
430
534
|
* Options the service cannot honour throw rather than being ignored. Silently
|
|
431
535
|
* dropping one would mean returning a result that does not match what was
|
|
@@ -434,6 +538,7 @@ class EuRedact {
|
|
|
434
538
|
*/
|
|
435
539
|
async redactViaCloud(text, options) {
|
|
436
540
|
const { CloudClient } = await Promise.resolve().then(() => __importStar(require("./cloud/client.js")));
|
|
541
|
+
const { CloudError } = await Promise.resolve().then(() => __importStar(require("./cloud/errors.js")));
|
|
437
542
|
const countries = options.countries ?? null;
|
|
438
543
|
if (!countries || countries.length !== 1) {
|
|
439
544
|
throw new Error('cloud mode needs exactly one country, e.g. { countries: ["BE"] }. The ' +
|
|
@@ -451,43 +556,37 @@ class EuRedact {
|
|
|
451
556
|
if (options.referentialIntegrity) {
|
|
452
557
|
throw new Error("referentialIntegrity is not supported in cloud mode");
|
|
453
558
|
}
|
|
454
|
-
// detectDates is deliberately NOT forwarded. The service always runs its
|
|
455
|
-
// rules engine with dates on, because that is what the model was trained
|
|
456
|
-
// against; the caller's value cannot change that. It is the one ignored
|
|
457
|
-
// option that is safe to ignore — it can only cause MORE to be detected,
|
|
458
|
-
// never less, so it cannot produce under-redaction.
|
|
459
559
|
const allowed = this.allowedFor(options);
|
|
460
560
|
const domains = this.domainsFor(options);
|
|
461
|
-
const
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
result.tokens = tokenMapper ? tokenMapper.tokens : {};
|
|
561
|
+
const client = new CloudClient();
|
|
562
|
+
// Dates are always on, whatever detectDates says: that is the rules output
|
|
563
|
+
// the model was trained against, and a date of birth the rules can place
|
|
564
|
+
// has no reason to travel. It is the one ignored option that is safe to
|
|
565
|
+
// ignore — it can only cause MORE to be detected, never less.
|
|
566
|
+
//
|
|
567
|
+
// No allowlist and no tokens on this pass. An exemption says what the
|
|
568
|
+
// caller wants back, not what may leave; both are applied below, after the
|
|
569
|
+
// response, to the merged spans.
|
|
570
|
+
const local = this.redactRules(text, { countries, detectDates: true, cache: options.cache }, new Map(), new Map());
|
|
571
|
+
const [masked, labels] = maskForCloud(text, local.detections);
|
|
572
|
+
// The client converts the service's code-point offsets to UTF-16 units of
|
|
573
|
+
// the text it sent, so everything from here on is in one unit.
|
|
574
|
+
const remote = await client.redact(masked, {
|
|
575
|
+
country: countries[0], idempotencyKey: options.idempotencyKey,
|
|
576
|
+
});
|
|
577
|
+
// A new array throughout: `local` may be the cached rules result.
|
|
578
|
+
const [detections, exempted] = applyAllowlist(text, [...local.detections, ...ontoOriginal(remote.detections, masked, text, labels, CloudError)], allowed, domains);
|
|
579
|
+
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
580
|
+
const tokenMapper = options.tokenize ? new TokenMapper(text, detections) : null;
|
|
581
|
+
return {
|
|
582
|
+
...local,
|
|
583
|
+
redactedText: applyReplacements(text, detections, this.labelFor(false, tokenMapper)),
|
|
584
|
+
detections,
|
|
585
|
+
source: "cloud",
|
|
586
|
+
tokens: tokenMapper ? tokenMapper.tokens : {},
|
|
587
|
+
exempted,
|
|
588
|
+
...(remote.cloud ? { cloud: remote.cloud } : {}),
|
|
589
|
+
};
|
|
491
590
|
}
|
|
492
591
|
redact(text, options = {}) {
|
|
493
592
|
checkCountryArg(options.countries, "countries");
|
|
@@ -508,12 +607,19 @@ class EuRedact {
|
|
|
508
607
|
if (requestedMode !== "rules") {
|
|
509
608
|
throw new Error(`unknown mode ${JSON.stringify(requestedMode)}: expected "rules" or "cloud"`);
|
|
510
609
|
}
|
|
610
|
+
if (options.idempotencyKey !== undefined) {
|
|
611
|
+
throw new Error('idempotencyKey applies to mode: "cloud" only: a rules-only redaction makes no request to deduplicate');
|
|
612
|
+
}
|
|
613
|
+
return this.redactRules(text, options, allowed, domains);
|
|
614
|
+
}
|
|
615
|
+
/** The local pipeline: normalise, detect, exempt, mask. */
|
|
616
|
+
redactRules(text, options, allowed, domains) {
|
|
511
617
|
if (text.length > this.maxInputLength) {
|
|
512
618
|
throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
|
|
513
619
|
`(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
|
|
514
620
|
`increase maxInputLength when constructing EuRedact.`);
|
|
515
621
|
}
|
|
516
|
-
const { countries = null, countryHint = null, context = null, chunkOffset = 0,
|
|
622
|
+
const { countries = null, countryHint = null, context = null, chunkOffset = 0, referentialIntegrity = false, tokenize = false, detectDates = false, } = options;
|
|
517
623
|
// A context makes the result depend on evidence from other chunks, so the
|
|
518
624
|
// text no longer identifies the result. Caching is disabled rather than
|
|
519
625
|
// keyed on the context, whose contents change as chunks arrive.
|
|
@@ -531,7 +637,7 @@ class EuRedact {
|
|
|
531
637
|
// separator, and two different lists must never share a key.
|
|
532
638
|
const allowKey = (allowed.size ? JSON.stringify([...allowed.keys()].sort()) : "")
|
|
533
639
|
+ (domains.size ? "|dom=" + JSON.stringify([...domains.keys()].sort()) : "");
|
|
534
|
-
const cacheMode =
|
|
640
|
+
const cacheMode = `rules|dates=${detectDates}|hint=${hintKey}|ri=${referentialIntegrity}|tok=${tokenize}|allow=${allowKey}`;
|
|
535
641
|
let cacheKey;
|
|
536
642
|
if (cache) {
|
|
537
643
|
cacheKey = this.cache.key(normalizedText, countriesTuple, cacheMode);
|
package/dist/cjs/types.js
CHANGED
|
@@ -79,7 +79,8 @@ var EntityType;
|
|
|
79
79
|
EntityType["BIOMETRIC_REF"] = "BIOMETRIC_REF";
|
|
80
80
|
EntityType["FINANCIAL_AMOUNT"] = "FINANCIAL_AMOUNT";
|
|
81
81
|
EntityType["QUASI_IDENTIFIER"] = "QUASI_IDENTIFIER";
|
|
82
|
-
|
|
82
|
+
/** Legacy alias of SECRET: every credential, the model's included, is SECRET (rules-engine#111). */
|
|
83
|
+
EntityType["CREDENTIAL"] = "SECRET";
|
|
83
84
|
EntityType["URL"] = "URL";
|
|
84
85
|
EntityType["OTHER"] = "OTHER";
|
|
85
86
|
})(EntityType || (exports.EntityType = EntityType = {}));
|
|
@@ -93,6 +94,8 @@ exports.LEGACY_TYPE_ALIASES = {
|
|
|
93
94
|
NAME: "PERSON_NAME",
|
|
94
95
|
STREET_ADDRESS: "ADDRESS",
|
|
95
96
|
NATIONALITY_ETHNICITY: "SENSITIVE_ATTRIBUTE",
|
|
97
|
+
// The model labels prose credentials CREDENTIAL; the SDK reports every credential as SECRET (rules-engine#111).
|
|
98
|
+
CREDENTIAL: "SECRET",
|
|
96
99
|
};
|
|
97
100
|
/**
|
|
98
101
|
* Resolve a possibly-legacy type name to its canonical form.
|
package/dist/esm/cache.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { sha256Hex } from "./platform.js";
|
|
2
2
|
// Default ceiling on retained characters, not entries. A count-only cap of
|
|
3
3
|
// 1024 entries says nothing about memory: 1024 cached 1 MB documents is a
|
|
4
4
|
// gigabyte of PII-bearing text held live. ~16M characters is a few tens of MB.
|
|
@@ -33,13 +33,7 @@ export class ResultCache {
|
|
|
33
33
|
// deployment an attacker who can guess a victim's document could submit a
|
|
34
34
|
// same-length colliding one and be served the victim's cached result,
|
|
35
35
|
// detections and raw matched PII included.
|
|
36
|
-
return
|
|
37
|
-
.update(text)
|
|
38
|
-
.update("|")
|
|
39
|
-
.update(sorted.join("|"))
|
|
40
|
-
.update("|")
|
|
41
|
-
.update(mode)
|
|
42
|
-
.digest("hex");
|
|
36
|
+
return sha256Hex([text, "|", sorted.join("|"), "|", mode]);
|
|
43
37
|
}
|
|
44
38
|
get(key) {
|
|
45
39
|
if (!this.enabled)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* [CLOUD EXTENSION] The account and job endpoints an API key may call.
|
|
3
|
+
*
|
|
4
|
+
* Everything a customer's key can do on the service is reachable from the SDK
|
|
5
|
+
* (rules-engine#89): redaction (`CloudClient`, `redactAsync`), batches
|
|
6
|
+
* (`Batches`), a past job (`Jobs`) and the account itself (`Account`). What
|
|
7
|
+
* needs a signed-in person -- logging in, minting keys, changing settings,
|
|
8
|
+
* members, accepting terms -- is not here, because a key cannot do it.
|
|
9
|
+
*
|
|
10
|
+
* Every typed value carries `raw`, the service's JSON, so a field the SDK does
|
|
11
|
+
* not know yet is still reachable.
|
|
12
|
+
*/
|
|
13
|
+
import { type CloudConfig } from "./config.js";
|
|
14
|
+
import type { RedactResult } from "../types.js";
|
|
15
|
+
type Json = Record<string, unknown>;
|
|
16
|
+
/** `GET /v1/account`: plan limits, retention and usage against quota. */
|
|
17
|
+
export interface AccountSummary {
|
|
18
|
+
tenantId: string | null;
|
|
19
|
+
tenantName: string | null;
|
|
20
|
+
active: boolean | null;
|
|
21
|
+
memberSince: string | null;
|
|
22
|
+
role: string | null;
|
|
23
|
+
/** `"ttl"`, or `"none"` when text is discarded on delivery. */
|
|
24
|
+
retentionMode: string | null;
|
|
25
|
+
payloadTtlHours: number | null;
|
|
26
|
+
documentsToday: number | null;
|
|
27
|
+
documentsThisMonth: number | null;
|
|
28
|
+
tokensThisMonth: number | null;
|
|
29
|
+
failures24h: number | null;
|
|
30
|
+
/** `null` for an account without a daily quota. */
|
|
31
|
+
dailyQuota: number | null;
|
|
32
|
+
quotaRemaining: number | null;
|
|
33
|
+
activeKeys: number | null;
|
|
34
|
+
batchesAvailable: boolean | null;
|
|
35
|
+
raw: Json;
|
|
36
|
+
}
|
|
37
|
+
/** `GET /v1/account/credits`: the balance, in credits. */
|
|
38
|
+
export interface Credits {
|
|
39
|
+
balance: number | null;
|
|
40
|
+
granted: number | null;
|
|
41
|
+
spent: number | null;
|
|
42
|
+
unit: string | null;
|
|
43
|
+
raw: Json;
|
|
44
|
+
}
|
|
45
|
+
/** One line of the credit history: movements rolled up per minute, reason and direction, newest first. */
|
|
46
|
+
export interface CreditEntry {
|
|
47
|
+
at: string | null;
|
|
48
|
+
/** Negative for a debit. */
|
|
49
|
+
delta: number | null;
|
|
50
|
+
reason: string | null;
|
|
51
|
+
count: number | null;
|
|
52
|
+
jobId: string | null;
|
|
53
|
+
sourceRef: string | null;
|
|
54
|
+
raw: Json;
|
|
55
|
+
}
|
|
56
|
+
/** One day of `GET /v1/account/usage`, or of one key's series. */
|
|
57
|
+
export interface UsageDay {
|
|
58
|
+
day: string | null;
|
|
59
|
+
documents: number | null;
|
|
60
|
+
tokens: number | null;
|
|
61
|
+
failures: number | null;
|
|
62
|
+
rulesOnly: number | null;
|
|
63
|
+
raw: Json;
|
|
64
|
+
}
|
|
65
|
+
/** One key of `GET /v1/account/usage/by-key`; the rest are rolled into one with `keyId` null. */
|
|
66
|
+
export interface KeyUsage {
|
|
67
|
+
keyId: number | null;
|
|
68
|
+
name: string | null;
|
|
69
|
+
documents: number | null;
|
|
70
|
+
tokens: number | null;
|
|
71
|
+
failures: number | null;
|
|
72
|
+
series: UsageDay[];
|
|
73
|
+
raw: Json;
|
|
74
|
+
}
|
|
75
|
+
/** One key of `GET /v1/account/keys`. Never its secret. */
|
|
76
|
+
export interface ApiKey {
|
|
77
|
+
id: number | null;
|
|
78
|
+
name: string | null;
|
|
79
|
+
createdBy: string | null;
|
|
80
|
+
createdAt: string | null;
|
|
81
|
+
lastUsedAt: string | null;
|
|
82
|
+
revokedAt: string | null;
|
|
83
|
+
active: boolean | null;
|
|
84
|
+
raw: Json;
|
|
85
|
+
}
|
|
86
|
+
/** The answer to `POST /v1/account/keys/{id}/revoke`. */
|
|
87
|
+
export interface KeyRevocation {
|
|
88
|
+
id: number | null;
|
|
89
|
+
keysRemaining: number | null;
|
|
90
|
+
/** True when no active key is left: nothing can call the service until a person mints one. */
|
|
91
|
+
lockedOut: boolean | null;
|
|
92
|
+
/** True when the key revoked is the one that made this call. */
|
|
93
|
+
selfRevoked: boolean | null;
|
|
94
|
+
raw: Json;
|
|
95
|
+
}
|
|
96
|
+
/** `GET /v1/jobs/{id}`: a job's state, and its result once it has one. */
|
|
97
|
+
export interface Job {
|
|
98
|
+
jobId: string;
|
|
99
|
+
/** `queued`, `running` or `succeeded`. A failed or rejected job throws instead. */
|
|
100
|
+
status: string;
|
|
101
|
+
createdAt: string | null;
|
|
102
|
+
/**
|
|
103
|
+
* Set when the job succeeded. Its text and offsets are those of the text the
|
|
104
|
+
* service received -- under the local-first cloud mode, the locally masked
|
|
105
|
+
* text, not the caller's original.
|
|
106
|
+
*/
|
|
107
|
+
result: RedactResult | null;
|
|
108
|
+
raw: Json;
|
|
109
|
+
}
|
|
110
|
+
export interface ApiClientOptions {
|
|
111
|
+
/** Injectable for tests. Defaults to the platform `fetch`. */
|
|
112
|
+
fetchImpl?: typeof fetch;
|
|
113
|
+
}
|
|
114
|
+
/** A JSON client: GETs are retried like the redact call; writes are sent once. */
|
|
115
|
+
declare class ApiClient {
|
|
116
|
+
readonly config: CloudConfig;
|
|
117
|
+
private readonly fetchImpl?;
|
|
118
|
+
constructor(config?: CloudConfig | null, options?: ApiClientOptions);
|
|
119
|
+
protected call(method: "GET" | "POST", path: string, params?: Record<string, number>, retry?: boolean): Promise<Json>;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* The account an API key belongs to: summary, credits, usage and keys.
|
|
123
|
+
*
|
|
124
|
+
* These calls pass with an API key while the service accepts account calls
|
|
125
|
+
* without a browser `Origin` (its `EUREDACT_CONSOLE_REQUIRE_ORIGIN=0`); a
|
|
126
|
+
* refusal arrives as a `CloudError` with status 403.
|
|
127
|
+
*/
|
|
128
|
+
export declare class Account extends ApiClient {
|
|
129
|
+
summary(): Promise<AccountSummary>;
|
|
130
|
+
credits(): Promise<Credits>;
|
|
131
|
+
/** Recent credit movements, newest first (1-500). */
|
|
132
|
+
creditHistory(limit?: number): Promise<CreditEntry[]>;
|
|
133
|
+
/** Documents, tokens and failures per day over the last `days`. */
|
|
134
|
+
usage(days?: number): Promise<UsageDay[]>;
|
|
135
|
+
/** The same window, split by the key that submitted the work. */
|
|
136
|
+
usageByKey(days?: number): Promise<KeyUsage[]>;
|
|
137
|
+
keys(): Promise<ApiKey[]>;
|
|
138
|
+
/**
|
|
139
|
+
* Revoke a key, e.g. one that leaked: the one write a key may make. Sent once,
|
|
140
|
+
* never retried. Already revoked throws a `CloudError` with status 409; a key
|
|
141
|
+
* that does not exist, or is another account's, throws `NotFoundError`.
|
|
142
|
+
* Revoking the key this client uses works, and is reported by `selfRevoked`.
|
|
143
|
+
*/
|
|
144
|
+
revokeKey(keyId: number): Promise<KeyRevocation>;
|
|
145
|
+
}
|
|
146
|
+
/** Past jobs, by id. */
|
|
147
|
+
export declare class Jobs extends ApiClient {
|
|
148
|
+
/**
|
|
149
|
+
* A job's state, and its result if it has one. Throws `ResultExpiredError`
|
|
150
|
+
* when the job succeeded but its result is no longer retained, and
|
|
151
|
+
* `NotFoundError` for an unknown job (or another account's).
|
|
152
|
+
*/
|
|
153
|
+
retrieve(jobId: string): Promise<Job>;
|
|
154
|
+
}
|
|
155
|
+
export {};
|