euredact 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -5
- package/dist/cjs/rules/countries/index.js +20 -3
- package/dist/cjs/rules/suppressors.js +200 -17
- package/dist/cjs/sdk.js +90 -7
- package/dist/esm/rules/countries/index.js +20 -3
- package/dist/esm/rules/suppressors.js +200 -17
- package/dist/esm/sdk.d.ts +26 -0
- package/dist/esm/sdk.js +90 -8
- package/dist/esm/types.d.ts +6 -4
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -82,7 +82,8 @@ console.log(result.detections);
|
|
|
82
82
|
| keep relationships visible across a whole session | `{ referentialIntegrity: true }` |
|
|
83
83
|
| never redact my own company name or addresses | `{ allowlist: [...] }`, or `new EuRedact({ allowlist })` |
|
|
84
84
|
| never redact anything at my own domain | `{ allowlistDomains: ["acme.be"] }` |
|
|
85
|
-
|
|
|
85
|
+
| leave whole types in the text (job titles, amounts) | `{ excludeTypes: ["JOB_TITLE", "FINANCIAL_AMOUNT"] }`, or `new EuRedact({ excludeTypes })` — see [Excluding entity types](#excluding-entity-types) |
|
|
86
|
+
| audit what the allowlist or `excludeTypes` kept in the document | `result.exempted` |
|
|
86
87
|
| catch person names, employers, job titles, diagnoses | `redactAsync(text, { mode: "cloud" })` — see [Cloud tier](#cloud-tier) |
|
|
87
88
|
| include dates of birth | `{ detectDates: true }` |
|
|
88
89
|
| redact a document too large for one call | `context` + `chunkOffset` — see [Chunked documents](#chunked-documents) |
|
|
@@ -112,6 +113,7 @@ interface RedactOptions {
|
|
|
112
113
|
tokenize?: boolean; // reversible tokens, EMAIL_K7Q2
|
|
113
114
|
allowlist?: string[] | null; // values never redacted
|
|
114
115
|
allowlistDomains?: string[] | null; // domains never redacted (EMAIL/URL)
|
|
116
|
+
excludeTypes?: string[] | null; // entity types neither reported nor masked
|
|
115
117
|
detectDates?: boolean; // include DOB / date of death
|
|
116
118
|
cache?: boolean; // reuse results for identical input
|
|
117
119
|
idempotencyKey?: string; // mode: "cloud" only; see the table below
|
|
@@ -136,6 +138,7 @@ interface RedactOptions {
|
|
|
136
138
|
| `tokenize` | `false` | Reversible token per value (`EMAIL_K7Q2`), with the mapping in `result.tokens` for [`restore()`](#restoretext-tokens). Unique to the **call**. Cannot be combined with `referentialIntegrity`. See [Reversible tokenization](#reversible-tokenization). |
|
|
137
139
|
| `allowlist` | `null` | Values never to redact, whole-span and case-insensitive, merged with the instance's list. Structured identifiers (IBAN, phone, VAT, …) also match across spacing and hyphenation. Applies to cloud-tier types too. See [Allowlist](#allowlist). |
|
|
138
140
|
| `allowlistDomains` | `null` | Domains whose addresses are never redacted, e.g. `["acme.be"]`. Applies to `EMAIL` and `URL` only, and covers subdomains. See [Allowlist](#allowlist). |
|
|
141
|
+
| `excludeTypes` | `null` | Entity types neither reported nor masked, e.g. `["JOB_TITLE"]`; merged with the instance's list. Unknown names throw. See [Excluding entity types](#excluding-entity-types). |
|
|
139
142
|
|
|
140
143
|
**Long documents and tiers**
|
|
141
144
|
|
|
@@ -229,6 +232,8 @@ The `EuRedact` class exposes `redact()`, `redactAsync()`, `redactBatch()`,
|
|
|
229
232
|
new EuRedact({
|
|
230
233
|
maxInputLength?: number, // default 10_485_760 (~10 MB)
|
|
231
234
|
allowlist?: string[] | null,
|
|
235
|
+
allowlistDomains?: string[] | null,
|
|
236
|
+
excludeTypes?: string[] | null,
|
|
232
237
|
})
|
|
233
238
|
```
|
|
234
239
|
|
|
@@ -237,6 +242,7 @@ new EuRedact({
|
|
|
237
242
|
| `maxInputLength` | `10_485_760` | Longest document `redact()` accepts, in characters. Above it, it throws — split the input or raise the ceiling. |
|
|
238
243
|
| `allowlist` | `null` | Values never to redact, for every call on this instance. Merged with the per-call `allowlist`. See [Allowlist](#allowlist). |
|
|
239
244
|
| `allowlistDomains` | `null` | Domains never to redact, for every call on this instance. |
|
|
245
|
+
| `excludeTypes` | `null` | Entity types never reported or masked, for every call on this instance. See [Excluding entity types](#excluding-entity-types). |
|
|
240
246
|
|
|
241
247
|
The result cache, referential-integrity labels and custom patterns are all per
|
|
242
248
|
instance, which is what makes one instance per tenant the right default.
|
|
@@ -259,7 +265,7 @@ interface RedactResult {
|
|
|
259
265
|
detectionMode: string; // "declared" if countries was passed,
|
|
260
266
|
// "inferred" otherwise
|
|
261
267
|
tokens: Record<string, string>; // token -> original value; only with tokenize: true
|
|
262
|
-
exempted: Exemption[]; // spans the allowlist kept, with the rule
|
|
268
|
+
exempted: Exemption[]; // spans the allowlist or excludeTypes kept, with the rule
|
|
263
269
|
cloud?: CloudInfo; // mode: "cloud" only: job id, model, cost, unplaced
|
|
264
270
|
// findings — see "What the service said"
|
|
265
271
|
}
|
|
@@ -1409,8 +1415,8 @@ sdk.redact("Mail jan@acme.be or piet@acme.be", { countries: ["NL"] }).redactedTe
|
|
|
1409
1415
|
label boundary, so `acme.be` does **not** exempt `evilacme.be`. Entries may be
|
|
1410
1416
|
written `acme.be`, `@acme.be` or `.acme.be`.
|
|
1411
1417
|
|
|
1412
|
-
There are deliberately **no wildcards**. The allowlist
|
|
1413
|
-
|
|
1418
|
+
There are deliberately **no wildcards**. The allowlist and `excludeTypes` are
|
|
1419
|
+
the only options that turn redaction *off*, so an over-broad entry fails toward under-redaction and
|
|
1414
1420
|
does so silently.
|
|
1415
1421
|
|
|
1416
1422
|
### What was exempted
|
|
@@ -1422,10 +1428,51 @@ for (const e of r.exempted) console.log(e.entityType, e.text, e.rule, e.ruleKind
|
|
|
1422
1428
|
```
|
|
1423
1429
|
|
|
1424
1430
|
`Exemption` carries `entityType`, `start`, `end`, `text`, the `rule` that
|
|
1425
|
-
matched as you wrote it, and `ruleKind` (`"value"` or `"
|
|
1431
|
+
matched as you wrote it, and `ruleKind` (`"value"`, `"domain"` or `"type"`). An exempted
|
|
1426
1432
|
span is absent from `detections`, so `exempted` is the only record that it was
|
|
1427
1433
|
found at all.
|
|
1428
1434
|
|
|
1435
|
+
## Excluding entity types
|
|
1436
|
+
|
|
1437
|
+
A deployment whose taxonomy is narrower than euRedact's can switch whole types
|
|
1438
|
+
off instead of filtering every result itself. Excluded types are neither in
|
|
1439
|
+
`detections` nor masked:
|
|
1440
|
+
|
|
1441
|
+
```ts
|
|
1442
|
+
const r = redact("Mail anna@acme.be or call +32 470 12 34 56.", {
|
|
1443
|
+
countries: ["BE"], excludeTypes: ["PHONE"],
|
|
1444
|
+
});
|
|
1445
|
+
r.redactedText;
|
|
1446
|
+
// 'Mail [EMAIL] or call +32 470 12 34 56.'
|
|
1447
|
+
for (const e of r.exempted) console.log(e.entityType, e.text, e.rule, e.ruleKind);
|
|
1448
|
+
// PHONE +32 470 12 34 56 PHONE type
|
|
1449
|
+
```
|
|
1450
|
+
|
|
1451
|
+
Set it once for a deployment with `new EuRedact({ excludeTypes: [...] })`; a per-call
|
|
1452
|
+
list is merged with it.
|
|
1453
|
+
|
|
1454
|
+
- **Nothing is excluded by default.** Every type is detected and redacted
|
|
1455
|
+
unless you name it.
|
|
1456
|
+
- **Names are checked.** They are case-insensitive, and legacy names resolve
|
|
1457
|
+
(`IBAN` is `BANK_ACCOUNT`, `CREDENTIAL` is `SECRET`). A registered custom
|
|
1458
|
+
pattern's name works too. An unknown name throws, so a typo
|
|
1459
|
+
cannot silently switch nothing off.
|
|
1460
|
+
- **It applies to each span's final type.** In
|
|
1461
|
+
`NHS Number: 5556953627` the digits also pass the Austrian SVNr check.
|
|
1462
|
+
Excluding `HEALTH_INSURANCE` leaves them in the text; it does not let them
|
|
1463
|
+
be masked as an `SSN` instead.
|
|
1464
|
+
- **Every excluded detection is recorded** in `result.exempted`, with
|
|
1465
|
+
`ruleKind: "type"` and the type's name as `rule`, so an audit can see what
|
|
1466
|
+
was deliberately left in.
|
|
1467
|
+
- **In `mode: "cloud"`** an excluded rules type is still masked in the text
|
|
1468
|
+
sent to the service. The option decides what comes back, not what may leave
|
|
1469
|
+
your machine. Model types (`JOB_TITLE`, `FINANCIAL_AMOUNT`, …) are filtered
|
|
1470
|
+
from the merged result.
|
|
1471
|
+
- **There is no include-list.** A list of types to keep would leave every type
|
|
1472
|
+
a later release adds unmasked for you, without notice.
|
|
1473
|
+
- **A type is one switch.** `INTERNAL_ID` covers employee and customer numbers
|
|
1474
|
+
as well as ticket and case numbers; excluding it leaves all of them in.
|
|
1475
|
+
|
|
1429
1476
|
## Architecture
|
|
1430
1477
|
|
|
1431
1478
|
```
|
|
@@ -4,6 +4,11 @@ exports.COUNTRY_CONFIGS = void 0;
|
|
|
4
4
|
const types_js_1 = require("../../types.js");
|
|
5
5
|
const validators_js_1 = require("../validators.js");
|
|
6
6
|
const suppressors_js_1 = require("../suppressors.js");
|
|
7
|
+
/** An assigned value: not opened by a quote or bracket, eight characters or
|
|
8
|
+
* more, not ended by punctuation or a closer (rules-engine#35, #126). */
|
|
9
|
+
const ASSIGNED = String.raw `[^\s"'\x60\[\]{}()<>„“”«»‚‘’]\S{6,}[^\s.,;:!?)'"\]}\x60“”»‘’]`;
|
|
10
|
+
/** A quoted assigned value: exactly what the quotes hold (rules-engine#126). */
|
|
11
|
+
const QUOTED_ASSIGNED = ['"', "'", "\x60"].flatMap((q) => [" ", ""].map((gap) => String.raw `(?<=[:=]` + gap + q + String.raw `)[^\s` + q + String.raw `]{8,}(?=` + q + ")")).join("|");
|
|
7
12
|
/** Day, month as a word, year ("4. März 1948"), behind a date label (rules-engine#115). */
|
|
8
13
|
function writtenDatePattern() {
|
|
9
14
|
const names = new Set(suppressors_js_1.MONTH_NAMES.split("|").filter(Boolean));
|
|
@@ -146,6 +151,7 @@ const DOB_CONTEXT = [
|
|
|
146
151
|
"γεννηθείς", "γεννήθηκε", "ημερομηνία γέννησης",
|
|
147
152
|
// West Slavic.
|
|
148
153
|
"urodzony", "urodzona", "data urodzenia",
|
|
154
|
+
"ur.",
|
|
149
155
|
"narozen", "datum narození", "narodený", "dátum narodenia",
|
|
150
156
|
// Hungarian, Romanian, Bulgarian.
|
|
151
157
|
"született", "születési", "născut", "născută", "data nașterii",
|
|
@@ -268,14 +274,20 @@ const SHARED = {
|
|
|
268
274
|
// AWS Secret Access Key (40 chars, base64 with +/ after an AKIA line)
|
|
269
275
|
p(types_js_1.EntityType.SECRET, String.raw `(?<=aws_secret_access_key\s{0,4}[:=]\s{0,4})[A-Za-z0-9+/]{40}`, null, "AWS Secret Access Key"),
|
|
270
276
|
// --- Secret / API Key (connection strings with embedded credentials) ---
|
|
271
|
-
|
|
277
|
+
// The tail stops at a quote (rules-engine#126).
|
|
278
|
+
p(types_js_1.EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]{1,256}:[^\s@]{1,256}@[^\s"'\x60]+`, null, "Connection string with credentials"),
|
|
272
279
|
// --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
|
|
273
280
|
// The final character may not be sentence punctuation. `\S` does not stop
|
|
274
281
|
// at a full stop, so "Reisepass: CA1234567." was claimed as the
|
|
275
282
|
// ten-character span "CA1234567." -- a passport number plus the sentence's
|
|
276
283
|
// period, masked as a credential because a "credentials" word sat within
|
|
277
284
|
// the context window (rules-engine#35).
|
|
278
|
-
|
|
285
|
+
// The value starts after an opening quote or bracket and ends before its
|
|
286
|
+
// closer: `user = "svc_terraform"` was the span `"svc_terraform`
|
|
287
|
+
// (rules-engine#126).
|
|
288
|
+
// A quoted value is what the quotes hold, closing punctuation included.
|
|
289
|
+
p(types_js_1.EntityType.SECRET, QUOTED_ASSIGNED + "|" + [String.raw `(?<=[:=] )`, String.raw `(?<=[:=])`, String.raw `(?<=[:=] ["'\x60\[{(])`, String.raw `(?<=[:=]["'\x60\[{(])`]
|
|
290
|
+
.map((lb) => lb + ASSIGNED).join("|"), "high_entropy", "Assigned secret value", SECRET_CONTEXT, true),
|
|
279
291
|
// --- Secret / API Key (entropy-based fallback for longer tokens) ---
|
|
280
292
|
// Only the *trailing* \b is replaced. It could not match when the token run
|
|
281
293
|
// ended on "-", "+" or "/", so the engine backtracked across two
|
|
@@ -555,7 +567,10 @@ const PT = {
|
|
|
555
567
|
p(types_js_1.EntityType.PHONE, String.raw `\b[29]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}\b`),
|
|
556
568
|
p(types_js_1.EntityType.PHONE, String.raw `\b[29]\d{8}\b`),
|
|
557
569
|
p(types_js_1.EntityType.PHONE, String.raw `\+351\s?[29]\d{2}[\s\-]?\d{3}[\s\-]?\d{3}`),
|
|
558
|
-
|
|
570
|
+
// The four formats Portugal has issued (rules-engine#126).
|
|
571
|
+
p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b(?:\d{2}-\d{2}-[A-Z]{2}|\d{2}-[A-Z]{2}-\d{2}|[A-Z]{2}-\d{2}-\d{2}|[A-Z]{2}-\d{2}-[A-Z]{2})\b`),
|
|
572
|
+
// Any three pairs, behind a plate label (rules-engine#126).
|
|
573
|
+
p(types_js_1.EntityType.LICENSE_PLATE, String.raw `\b[A-Z0-9]{2}-[A-Z0-9]{2}-[A-Z0-9]{2}\b`, null, "Portuguese license plate — AA-XX-XX, behind its label", ["matrícula", "matricula", "placa", "license plate", "number plate", "plate", "Kennzeichen", "kenteken", "immatriculation", "targa"], true),
|
|
559
574
|
p(types_js_1.EntityType.POSTAL_CODE, String.raw `\b\d{4}-\d{3}\b`),
|
|
560
575
|
],
|
|
561
576
|
};
|
|
@@ -630,6 +645,8 @@ const UK = {
|
|
|
630
645
|
code: "UK",
|
|
631
646
|
name: "United Kingdom",
|
|
632
647
|
patterns: [
|
|
648
|
+
// A sort code behind its label; no longer found through the Portuguese plate shape (rules-engine#126).
|
|
649
|
+
p(types_js_1.EntityType.BANK_ACCOUNT, String.raw `\b\d{2}-\d{2}-\d{2}\b`, null, "UK sort code — NN-NN-NN, behind its label", ["sort code", "sortcode", "sort-code"], true),
|
|
633
650
|
// Healthcare-provider numbers, behind their label: no shape of their own, so without a pattern nothing claimed them, or a phone or a foreign checksum did (canon HEALTHCARE_PROVIDER; rules-engine#111).
|
|
634
651
|
p(types_js_1.EntityType.HEALTHCARE_PROVIDER, String.raw `\b\d{7}\b`, null, "UK GMC number — 7 digits", ["GMC number", "GMC no", "GMC reference", "GMC reg", "GMC:", "GMC-Nr", "General Medical Council"], true),
|
|
635
652
|
// UK NINO: a social-security number, so SSN, as the canon defaults it (rules-engine#47).
|
|
@@ -203,6 +203,27 @@ function suppressSecretOverStructured(text, match) {
|
|
|
203
203
|
* natural word. A random all-letter token would be mixed case and is left alone
|
|
204
204
|
* — losing a real secret is the expensive direction.
|
|
205
205
|
*/
|
|
206
|
+
/**
|
|
207
|
+
* Values an assigned-secret rule finds after a "=" or ":" that are names, not
|
|
208
|
+
* secrets (rules-engine#126): `${VAR}` references, identifiers, hyphenated
|
|
209
|
+
* compounds, image tags, product versions, code calls. Not paths, host names
|
|
210
|
+
* or reference numbers: those carry personal data ("mac-mhartmann88").
|
|
211
|
+
* Kept in sync with the Python SDK's _SECRET_NAME_SHAPES.
|
|
212
|
+
*/
|
|
213
|
+
const SECRET_NAME_SHAPES = new RegExp([
|
|
214
|
+
String.raw `^\$\{[A-Za-z_][A-Za-z0-9_]*(?::?[-=?+][^}]*)?\}?$`,
|
|
215
|
+
String.raw `^[a-z]{2,}(?:_[a-z]{2,})+$`, String.raw `^[A-Z]{2,}(?:_[A-Z]{2,})+$`,
|
|
216
|
+
String.raw `^(?:[A-Z]+[a-z]{2,}){3,}(?:V?\d{1,2})?$`,
|
|
217
|
+
String.raw `^[a-z][a-z0-9._/-]*:(?:\d+(?:\.\d+)*(?:-[a-z0-9]+)*|latest|stable|alpine|slim)$`,
|
|
218
|
+
String.raw `^[A-Za-z][\w.-]*\/\d+(?:\.\d+)+$`,
|
|
219
|
+
String.raw `^[\w.]+\(["'\x60]`,
|
|
220
|
+
].join("|"));
|
|
221
|
+
/** A hyphenated compound of words and acronyms; two capitalised words alone
|
|
222
|
+
* ("Ann-Sophie", "Caron-Cordier") are names and are left alone. */
|
|
223
|
+
const COMPOUND_PART = String.raw `(?:[A-ZÀ-Þ]{2,6}\d{0,3}|[A-Za-zÀ-ÿ][a-zß-ÿ]{2,})`;
|
|
224
|
+
const COMPOUND = new RegExp(String.raw `^` + COMPOUND_PART + String.raw `(?:-` + COMPOUND_PART + String.raw `)+$`);
|
|
225
|
+
const NAME_PAIR = /^[A-ZÀ-Þ][a-zß-ÿ]{2,}-[A-ZÀ-Þ][a-zß-ÿ]{2,}$/;
|
|
226
|
+
const CONNECTION_WITH_REFERENCE = /^[a-z]+:\/\/[^\s:@]+:\$\{[^}]*\}@/;
|
|
206
227
|
function suppressSecretNotASecret(_text, match) {
|
|
207
228
|
if (match.patternDef.entityType !== types_js_1.EntityType.SECRET)
|
|
208
229
|
return false;
|
|
@@ -217,6 +238,10 @@ function suppressSecretNotASecret(_text, match) {
|
|
|
217
238
|
return true;
|
|
218
239
|
if (HYPHENATED_WORDS.test(bare))
|
|
219
240
|
return true;
|
|
241
|
+
if (SECRET_NAME_SHAPES.test(bare) || CONNECTION_WITH_REFERENCE.test(bare))
|
|
242
|
+
return true;
|
|
243
|
+
if (COMPOUND.test(bare) && !NAME_PAIR.test(bare))
|
|
244
|
+
return true;
|
|
220
245
|
const alpha = /^[A-Za-zÀ-ÿ]+$/.test(bare);
|
|
221
246
|
const oneWord = bare === bare.toLowerCase() ||
|
|
222
247
|
(bare.slice(0, 1) === bare.slice(0, 1).toUpperCase() &&
|
|
@@ -358,7 +383,20 @@ function suppressSequential(_text, match) {
|
|
|
358
383
|
// the "]". The opening bracket is still required.
|
|
359
384
|
const PLACEHOLDER = /^(?:\[([A-Z][A-Z0-9_]*)\]?|([A-Z][A-Z0-9_]*?)_(?:\d+|[A-HJ-NP-Z2-9]{4}))$/;
|
|
360
385
|
const KNOWN_TYPE_NAMES = new Set(Object.values(types_js_1.EntityType));
|
|
361
|
-
function suppressRedactionPlaceholder(
|
|
386
|
+
function suppressRedactionPlaceholder(text, match) {
|
|
387
|
+
// The name between the brackets: an assigned value now starts after an
|
|
388
|
+
// opening bracket, so "credentials: [POSTAL_CODE]" offers "POSTAL_CODE" alone
|
|
389
|
+
// (rules-engine#33, #126).
|
|
390
|
+
if (text[match.start - 1] === "[" && text[match.end] === "]" && KNOWN_TYPE_NAMES.has(match.text))
|
|
391
|
+
return true;
|
|
392
|
+
// A placeholder inside the span: on text masked once, a value running across
|
|
393
|
+
// "[IP_ADDRESS]" is two things joined (rules-engine#126).
|
|
394
|
+
// One character either side: a span may start just inside the bracket.
|
|
395
|
+
const around = text.slice(Math.max(0, match.start - 1), match.end + 1);
|
|
396
|
+
for (const m of around.matchAll(/\[([A-Z][A-Z0-9_]*)\]/g)) {
|
|
397
|
+
if (KNOWN_TYPE_NAMES.has(m[1]))
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
362
400
|
const found = PLACEHOLDER.exec(match.text);
|
|
363
401
|
if (found === null)
|
|
364
402
|
return false;
|
|
@@ -506,6 +544,33 @@ const EXTENSION_BEFORE = /(?<![A-Za-z0-9_])(?:toest(?:el)?|ext|extension|poste|d
|
|
|
506
544
|
*/
|
|
507
545
|
const COUNT_LABEL_BEFORE = /(?<![A-Za-z0-9_])(?:employees|headcount|staff|mitarbeiter(?:innen)?|beschäftigte|werknemers|medewerkers|employés|salariés|effectifs?|dipendenti|empleados|pracowników|zaměstnanců)\s*[:=]?\s*$/i;
|
|
508
546
|
const NOT_POSTAL_BEFORE = /(?<![A-Za-z0-9_])(?:(?:ISO(?:\/IEC)?|IEC|EN|DIN(?:\s+EN)?|NEN(?:-EN)?|NF\s+EN|BS(?:\s+EN)?|PN-EN|ÖNORM)[\s\-]*|[A-Za-z]*[a-z][A-Z][A-Za-z]*\s+)$/;
|
|
547
|
+
/**
|
|
548
|
+
* What else a number is, by what touches it (rules-engine#126): a build,
|
|
549
|
+
* version, event, lease, policy or ticket number; a port; an invoice, reference
|
|
550
|
+
* or study-subject number; a process id; a JSON id or port field; an HTTP
|
|
551
|
+
* response size; the hour after a date; a lab reference range; or a quantity
|
|
552
|
+
* with its unit after it. Kept in sync with the Python SDK.
|
|
553
|
+
*/
|
|
554
|
+
const TECH_WORD_BEFORE = /(?<![A-Za-z0-9_])(?:build|version|policy\s+id|ticket(?:\s+[A-Za-z]{2,10})?|event|lease|pid|job|port|rfc|facture|invoice|r[ée]f[ée]rence|n[°º]|tn|sae|uczestnik|subject|narratief|participant)\.?\s*[:#=]?\s*["(«[]?\s*$/i;
|
|
555
|
+
const TECH_SHAPE_BEFORE = new RegExp("(?:" + [
|
|
556
|
+
String.raw `"[A-Za-z_]*(?:id|port)"\s*:\s*"?`, // "parent_id": 4400
|
|
557
|
+
String.raw `(?<![A-Za-z0-9])[A-Z_]*PORT\s*[:=]\s*"?`, // DB_PORT: "5432"
|
|
558
|
+
String.raw `(?:localhost|[a-z0-9\-]+\.[a-z0-9.\-]+):`, // localhost:3000
|
|
559
|
+
String.raw `(?<![\d.])\d{2,5}:`, // "8080:8080"
|
|
560
|
+
String.raw `HTTP\/\d(?:\.\d)?"?\s+\d{3}\s+`, // HTTP/1.1" 200 2048
|
|
561
|
+
String.raw `\d+\.\d+(?:\.\d+)+\s*\(`, // build 2.4.0 (4012)
|
|
562
|
+
String.raw `(?<!\d)\d{1,2}[./]\d{1,2}[./]`, // 03.10.2026 23:50
|
|
563
|
+
String.raw `\((?:N|norm(?:al)?|ref)\s*:?\s*`, // (N : 30-400)
|
|
564
|
+
String.raw `\|`,
|
|
565
|
+
].join("|") + ")$", "i");
|
|
566
|
+
const POSTAL_UNIT_AFTER = /^(?:\s?(?:ms|s|sec|secs|seconds?|secondes?|sekunden|sek|min|bits?|bytes?|kb|mb|gb|tb|kbps|mbps|px|%|records|rows|entries|datensätze|einträge|micrograms|µg|ug|mg|mmol|fl)(?![A-Za-zÀ-ÿ])|\s*→\s*\d+\s*(?:bits?|bytes?)(?![A-Za-z])|:\d|\|)/i;
|
|
567
|
+
const PID_BEFORE = /[A-Za-z0-9_\-]\[$/;
|
|
568
|
+
/** A standard's name as the letters of a lettered postcode: "RFC 5737". Not
|
|
569
|
+
* BS or EN: those are Bristol and Enfield. */
|
|
570
|
+
const STANDARD_LETTERS = /^(?:RFC|ISO|IEC|DIN)\s?\d/;
|
|
571
|
+
/** A year closing a range: "1998–2011". En and em dashes only. */
|
|
572
|
+
const YEAR_RANGE_BEFORE = /(?:19|20)\d\d\s?[–—]\s?$/;
|
|
573
|
+
const YEAR_RANGE_AFTER = /^\s?[–—]\s?(?:19|20)\d\d(?!\d)/;
|
|
509
574
|
/** A thousands group of an amount: the "750" of "£28,750,000". */
|
|
510
575
|
const GROUP_BEFORE = /\d[,.]$/;
|
|
511
576
|
const GROUP_AFTER = /^[,.]\d{3}(?!\d)/;
|
|
@@ -519,10 +584,90 @@ function suppressPostalAfterDisqualifyingWord(text, match) {
|
|
|
519
584
|
return true;
|
|
520
585
|
if (GROUP_BEFORE.test(before) && GROUP_AFTER.test(text.slice(match.end, match.end + 5)))
|
|
521
586
|
return true;
|
|
587
|
+
const after = text.slice(match.end, match.end + 24);
|
|
588
|
+
if (TECH_WORD_BEFORE.test(before) || TECH_SHAPE_BEFORE.test(before) || POSTAL_UNIT_AFTER.test(after)
|
|
589
|
+
|| STANDARD_LETTERS.test(match.text))
|
|
590
|
+
return true;
|
|
591
|
+
if (PID_BEFORE.test(before) && after.startsWith("]"))
|
|
592
|
+
return true;
|
|
593
|
+
if (DATE_YEAR.test(match.text.trim()) && (YEAR_RANGE_BEFORE.test(before) || YEAR_RANGE_AFTER.test(after)))
|
|
594
|
+
return true;
|
|
522
595
|
return RECENT_YEAR.test(match.text.trim())
|
|
523
596
|
&& (YEAR_WORD_BEFORE.test(before) || YEAR_VERB_BEFORE.test(before))
|
|
524
597
|
&& !TOWN_AFTER.test(text.slice(match.end, match.end + 3));
|
|
525
598
|
}
|
|
599
|
+
/** A group with a leading zero: "2.00.079.01" is a version, not an address. */
|
|
600
|
+
const IP_LEADING_ZERO = /(?:^|\.)0\d/;
|
|
601
|
+
const CIDR_PREFIX_AFTER = /^\/(\d{1,2})(?!\d)/;
|
|
602
|
+
/** RFC 5737's documentation ranges: reserved, never routed, used in examples. */
|
|
603
|
+
const DOCUMENTATION_NETS = ["192.0.2.", "198.51.100.", "203.0.113."];
|
|
604
|
+
/**
|
|
605
|
+
* An IPv4 shape that names no host (rules-engine#126): a network address
|
|
606
|
+
* ("10.40.0.0/16"), a documentation address, or a version with leading zeros.
|
|
607
|
+
* An interface address with its prefix ("192.168.1.23/24") stays.
|
|
608
|
+
*/
|
|
609
|
+
function suppressIpNotAHost(text, match) {
|
|
610
|
+
if (match.patternDef.entityType !== types_js_1.EntityType.IP_ADDRESS)
|
|
611
|
+
return false;
|
|
612
|
+
const ip = match.text;
|
|
613
|
+
if (IP_LEADING_ZERO.test(ip) || DOCUMENTATION_NETS.some((n) => ip.startsWith(n)))
|
|
614
|
+
return true;
|
|
615
|
+
// Part of a longer dotted run: "urn:oid:1.2.250.1.71.4.2.1" is an OID.
|
|
616
|
+
if (/\d\.$/.test(text.slice(Math.max(0, match.start - 2), match.start))
|
|
617
|
+
|| /^\.\d/.test(text.slice(match.end, match.end + 2)))
|
|
618
|
+
return true;
|
|
619
|
+
const prefix = CIDR_PREFIX_AFTER.exec(text.slice(match.end, match.end + 4));
|
|
620
|
+
if (prefix !== null && Number(prefix[1]) < 32) {
|
|
621
|
+
const value = ip.split(".").reduce((acc, o) => acc * 256 + Number(o), 0);
|
|
622
|
+
return value % 2 ** (32 - Number(prefix[1])) === 0;
|
|
623
|
+
}
|
|
624
|
+
return false;
|
|
625
|
+
}
|
|
626
|
+
/**
|
|
627
|
+
* The specimen of a form: "das Muster „C01X00T47“". Only "Muster": "z. B." and
|
|
628
|
+
* "e.g." introduce real values in prose as often as samples (rules-engine#126).
|
|
629
|
+
*/
|
|
630
|
+
const EXAMPLE_BEFORE = /(?<![A-Za-z])muster\s*[:,]?\s*[„"«']?$/i;
|
|
631
|
+
/** A dummy value in placeholder glyphs ("XX0000000", "999 999 9999") or MAC
|
|
632
|
+
* groups of one repeated letter. Not "111222333", the standard test BSN. */
|
|
633
|
+
const DUMMY_VALUE = /^(?=(?:[\s:\-./]*[X09]){5})(?:[X\s:\-./]*[X][X0\s:\-./]*|[X9\s:\-./]*X[X9\s:\-./]*|[0\s:\-./]+|[9\s:\-./]+)$|^(?:([a-fA-F])\1[:\-]){2,}([a-fA-F])\2$/;
|
|
634
|
+
/** Algorithm names in a key or signature line: "ED25519", "SHA256". */
|
|
635
|
+
const ALGORITHM = /^(?:ED25519|ED448|SHA-?(?:1|224|256|384|512)|MD5|AES-?\d{3}|RSA-?\d{3,4}|[HRE]S(?:256|384|512))$/i;
|
|
636
|
+
/** A date exactly, which no national number is written as. */
|
|
637
|
+
const EXACT_DOTTED_DATE = /^(?:0[1-9]|[12]\d|3[01])\.(?:0[1-9]|1[0-2])\.(?:19|20)\d\d$/;
|
|
638
|
+
/** The identifier types an example marker or a dummy value rules out. */
|
|
639
|
+
const EXAMPLE_TYPES = new Set([
|
|
640
|
+
types_js_1.EntityType.NATIONAL_ID, types_js_1.EntityType.PASSPORT, types_js_1.EntityType.LICENSE_PLATE,
|
|
641
|
+
types_js_1.EntityType.HEALTH_INSURANCE, types_js_1.EntityType.MAC_ADDRESS,
|
|
642
|
+
types_js_1.EntityType.DRIVERS_LICENSE, types_js_1.EntityType.SSN, types_js_1.EntityType.TAX_ID,
|
|
643
|
+
]);
|
|
644
|
+
/** A placeholder in the shape of a value, for any type (#126). */
|
|
645
|
+
function suppressDummyValue(_text, match) {
|
|
646
|
+
return DUMMY_VALUE.test(match.text.trim());
|
|
647
|
+
}
|
|
648
|
+
/** An identifier given as a form's specimen, not anyone's (#126). */
|
|
649
|
+
function suppressExampleValue(text, match) {
|
|
650
|
+
if (!EXAMPLE_TYPES.has(match.patternDef.entityType))
|
|
651
|
+
return false;
|
|
652
|
+
return EXAMPLE_BEFORE.test(text.slice(Math.max(0, match.start - 24), match.start));
|
|
653
|
+
}
|
|
654
|
+
/**
|
|
655
|
+
* Shapes that fit an identifier pattern and are something else (#126): a
|
|
656
|
+
* passport without a digit, an algorithm or a law citation read as a plate,
|
|
657
|
+
* a date read as a dotted national number.
|
|
658
|
+
*/
|
|
659
|
+
function suppressNotAnIdentifier(text, match) {
|
|
660
|
+
const type = match.patternDef.entityType;
|
|
661
|
+
const value = match.text.trim();
|
|
662
|
+
if (type === types_js_1.EntityType.PASSPORT)
|
|
663
|
+
return !/\d/.test(value);
|
|
664
|
+
if (type === types_js_1.EntityType.LICENSE_PLATE) {
|
|
665
|
+
return ALGORITHM.test(value) || /§\s*$/.test(text.slice(Math.max(0, match.start - 4), match.start));
|
|
666
|
+
}
|
|
667
|
+
if (type === types_js_1.EntityType.NATIONAL_ID)
|
|
668
|
+
return EXACT_DOTTED_DATE.test(value);
|
|
669
|
+
return false;
|
|
670
|
+
}
|
|
526
671
|
function suppressPhoneServiceNumber(_text, match) {
|
|
527
672
|
if (match.patternDef.entityType !== types_js_1.EntityType.PHONE)
|
|
528
673
|
return false;
|
|
@@ -634,7 +779,11 @@ function suppressDePlateUnknownDistrict(text, match) {
|
|
|
634
779
|
const parts = match.text.trim().split(/[\s\-]+/);
|
|
635
780
|
if (parts.length === 0 || !parts[0])
|
|
636
781
|
return false;
|
|
637
|
-
|
|
782
|
+
// A German plate number never starts with 0: without a plate cue,
|
|
783
|
+
// "GR-NB-0412" is an asset tag and "HR-IT-03" a form (rules-engine#126).
|
|
784
|
+
const digits = /\d+/.exec(match.text);
|
|
785
|
+
const leadingZero = digits !== null && digits[0].startsWith("0");
|
|
786
|
+
if (deDistricts_js_1.DE_DISTRICT_CODES.has(parts[0].toUpperCase()) && !leadingZero)
|
|
638
787
|
return false;
|
|
639
788
|
const [before, after] = getContext(text, match.start, match.end);
|
|
640
789
|
if (PLATE_CUE_NEAR.test(before + after))
|
|
@@ -1076,15 +1225,22 @@ function suppressRequiresContext(text, match) {
|
|
|
1076
1225
|
return false;
|
|
1077
1226
|
// A value filling an entire field of a delimited row is introduced by its
|
|
1078
1227
|
// column, not by a word, so that is context — but only for narrow shapes.
|
|
1228
|
+
// A date's column decides when the table names it: under "début" or
|
|
1229
|
+
// "last_day" a date is not a birth date (rules-engine#126).
|
|
1079
1230
|
if (DELIMITED_FIELD_TYPES.has(match.patternDef.entityType) &&
|
|
1080
|
-
fillsADelimitedField(text, match.start, match.end)
|
|
1231
|
+
fillsADelimitedField(text, match.start, match.end) &&
|
|
1232
|
+
!(DATE_LABEL_TYPES.has(match.patternDef.entityType) && columnHeader(text, match.start) !== null))
|
|
1081
1233
|
return false;
|
|
1082
1234
|
if (DATE_LABEL_TYPES.has(match.patternDef.entityType)) {
|
|
1083
1235
|
return !labelsThisDate(text, match.start, match.end, match.patternDef.contextKeywords, match.patternDef.entityType);
|
|
1084
1236
|
}
|
|
1085
1237
|
const [before, after] = getContext(text, match.start, match.end);
|
|
1086
1238
|
const context = (before + " " + after).toLowerCase();
|
|
1087
|
-
|
|
1239
|
+
if (match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase())))
|
|
1240
|
+
return false;
|
|
1241
|
+
// The column names it: a passport under "Passport" rows below the header (#130).
|
|
1242
|
+
const header = columnHeader(text, match.start);
|
|
1243
|
+
return header === null || !match.patternDef.contextKeywords.some(kw => keywordAt(header, kw, false) >= 0);
|
|
1088
1244
|
}
|
|
1089
1245
|
/** Date types decided by a label: DOB and DATE_OF_DEATH share one date shape. */
|
|
1090
1246
|
const DATE_LABEL_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
|
|
@@ -1120,10 +1276,16 @@ function nothingButAListBetween(gap) {
|
|
|
1120
1276
|
*/
|
|
1121
1277
|
function keywordAt(haystack, keyword, last) {
|
|
1122
1278
|
const kw = keyword.toLowerCase();
|
|
1279
|
+
const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
|
|
1123
1280
|
if (keyword !== keyword.toUpperCase() || keyword === kw) {
|
|
1124
|
-
|
|
1281
|
+
// It must start a word, though it may run on into a longer one: "née le"
|
|
1282
|
+
// in "signée le" and "born" in "Newborn" are other words (rules-engine#126).
|
|
1283
|
+
let i = last ? haystack.lastIndexOf(kw) : haystack.indexOf(kw);
|
|
1284
|
+
while (i > 0 && isLetter(haystack[i - 1])) {
|
|
1285
|
+
i = last ? (i === 0 ? -1 : haystack.lastIndexOf(kw, i - 1)) : haystack.indexOf(kw, i + 1);
|
|
1286
|
+
}
|
|
1287
|
+
return i;
|
|
1125
1288
|
}
|
|
1126
|
-
const isLetter = (c) => c !== "" && c.toLowerCase() !== c.toUpperCase();
|
|
1127
1289
|
const hits = [];
|
|
1128
1290
|
for (let i = haystack.indexOf(kw); i !== -1; i = haystack.indexOf(kw, i + 1)) {
|
|
1129
1291
|
let next = i + kw.length;
|
|
@@ -1185,29 +1347,44 @@ function columnHeader(text, start) {
|
|
|
1185
1347
|
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
1186
1348
|
const lineEndAt = text.indexOf("\n", start);
|
|
1187
1349
|
const line = text.slice(lineStart, lineEndAt >= 0 ? lineEndAt : text.length);
|
|
1188
|
-
// The three lines above, by lastIndexOf: splitting the whole prefix was
|
|
1189
|
-
// quadratic in the document, once per date candidate.
|
|
1190
|
-
let aboveStart = lineStart - 1;
|
|
1191
|
-
for (let k = 0; k < 3 && aboveStart >= 0; k++) {
|
|
1192
|
-
aboveStart = aboveStart === 0 ? -1 : text.lastIndexOf("\n", aboveStart - 1);
|
|
1193
|
-
}
|
|
1194
|
-
const above = lineStart ? text.slice(aboveStart + 1, Math.max(0, lineStart - 1)).split("\n") : [];
|
|
1195
1350
|
for (const sep of COLUMN_SEPARATORS) {
|
|
1196
1351
|
if (!line.includes(sep))
|
|
1197
1352
|
continue;
|
|
1198
1353
|
const cells = line.split(sep);
|
|
1199
1354
|
const column = text.slice(lineStart, start).split(sep).length - 1;
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1355
|
+
// Lines above, one lastIndexOf at a time: one of the three lines above, or
|
|
1356
|
+
// further up through rows of the same table (rules-engine#126).
|
|
1357
|
+
let end = lineStart - 1;
|
|
1358
|
+
let seen = 0;
|
|
1359
|
+
while (end >= 0 && seen < HEADER_REACH) {
|
|
1360
|
+
const begin = end === 0 ? 0 : text.lastIndexOf("\n", end - 1) + 1;
|
|
1361
|
+
const header = text.slice(begin, end);
|
|
1362
|
+
seen++;
|
|
1363
|
+
// A rule line ("|---|---|", "----+----") is neither header nor row (#130).
|
|
1364
|
+
if (RULE_LINE.test(header)) {
|
|
1365
|
+
end = begin - 1;
|
|
1203
1366
|
continue;
|
|
1367
|
+
}
|
|
1204
1368
|
const heads = header.split(sep);
|
|
1205
|
-
|
|
1369
|
+
const same = heads.length === cells.length;
|
|
1370
|
+
// A "label : value" field with a comma in its value is not a header
|
|
1371
|
+
// (#111); a commented header ("# Spalten: Name;von;bis") is (#126).
|
|
1372
|
+
// Nor is a data row without a date: a header has no cell with four digits or more (#130).
|
|
1373
|
+
if (same && (!header.includes(":") || header.trimStart().startsWith("#")) && !ANY_DATE.test(header)
|
|
1374
|
+
&& !heads.some((h) => (h.match(/\d/g) ?? []).length >= 4)) {
|
|
1206
1375
|
return heads[column].toLowerCase();
|
|
1376
|
+
}
|
|
1377
|
+
if (seen >= 3 && !same)
|
|
1378
|
+
break; // past the table this value sits in
|
|
1379
|
+
end = begin - 1;
|
|
1207
1380
|
}
|
|
1208
1381
|
}
|
|
1209
1382
|
return null;
|
|
1210
1383
|
}
|
|
1384
|
+
/** How many lines up a table's header is looked for. */
|
|
1385
|
+
const HEADER_REACH = 200;
|
|
1386
|
+
/** A table's rule line: only dashes, equals signs, plus signs, colons, pipes. */
|
|
1387
|
+
const RULE_LINE = /^[\s\-=+:|]*-[\s\-=+:|]*$/;
|
|
1211
1388
|
/** A dotted MAC shape glued to more of a reference is part of it: the tail of
|
|
1212
1389
|
* "WSO-II.6151.4471.2025" (rules-engine#105). Dotted form only. */
|
|
1213
1390
|
function suppressMacInReference(text, match) {
|
|
@@ -1242,12 +1419,18 @@ const TYPE_SUPPRESSORS = {
|
|
|
1242
1419
|
[types_js_1.EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
1243
1420
|
[types_js_1.EntityType.MAC_ADDRESS]: [suppressMacInReference],
|
|
1244
1421
|
[types_js_1.EntityType.CREDIT_CARD]: [suppressCardInsideDigitRun],
|
|
1422
|
+
[types_js_1.EntityType.IP_ADDRESS]: [suppressIpNotAHost],
|
|
1245
1423
|
};
|
|
1424
|
+
for (const type of EXAMPLE_TYPES) {
|
|
1425
|
+
TYPE_SUPPRESSORS[type] = [...(TYPE_SUPPRESSORS[type] ?? []), suppressExampleValue, suppressNotAnIdentifier];
|
|
1426
|
+
}
|
|
1246
1427
|
function shouldSuppress(text, match, scratch) {
|
|
1247
1428
|
if (suppressSequential(text, match))
|
|
1248
1429
|
return true;
|
|
1249
1430
|
if (suppressRedactionPlaceholder(text, match))
|
|
1250
1431
|
return true;
|
|
1432
|
+
if (suppressDummyValue(text, match))
|
|
1433
|
+
return true;
|
|
1251
1434
|
const typeSups = TYPE_SUPPRESSORS[match.patternDef.entityType];
|
|
1252
1435
|
if (typeSups) {
|
|
1253
1436
|
for (const s of typeSups) {
|