euredact 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +189 -0
- package/README.md +339 -10
- package/dist/cjs/cache.js +56 -0
- package/dist/cjs/index.js +34 -0
- package/dist/cjs/normalizer.js +45 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/rules/countries/index.js +628 -0
- package/dist/cjs/rules/engine.js +129 -0
- package/dist/cjs/rules/matchers.js +56 -0
- package/dist/cjs/rules/structural.js +104 -0
- package/dist/cjs/rules/suppressors.js +211 -0
- package/dist/cjs/rules/validators.js +699 -0
- package/dist/cjs/sdk.js +80 -0
- package/dist/cjs/types.js +42 -0
- package/dist/esm/cache.d.ts +11 -0
- package/dist/esm/cache.js +52 -0
- package/dist/esm/index.d.ts +10 -0
- package/dist/esm/index.js +23 -0
- package/dist/esm/normalizer.d.ts +2 -0
- package/dist/esm/normalizer.js +41 -0
- package/dist/esm/rules/countries/index.d.ts +2 -0
- package/dist/esm/rules/countries/index.js +625 -0
- package/dist/esm/rules/engine.d.ts +15 -0
- package/dist/esm/rules/engine.js +125 -0
- package/dist/esm/rules/matchers.d.ts +12 -0
- package/dist/esm/rules/matchers.js +52 -0
- package/dist/esm/rules/structural.d.ts +2 -0
- package/dist/esm/rules/structural.js +101 -0
- package/dist/esm/rules/suppressors.d.ts +9 -0
- package/dist/esm/rules/suppressors.js +208 -0
- package/dist/esm/rules/validators.d.ts +43 -0
- package/dist/esm/rules/validators.js +654 -0
- package/dist/esm/sdk.d.ts +16 -0
- package/dist/esm/sdk.js +76 -0
- package/dist/esm/types.d.ts +65 -0
- package/dist/esm/types.js +39 -0
- package/package.json +55 -9
- package/index.js +0 -3
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { DetectionSource } from "../types.js";
|
|
2
|
+
import { MultiPatternMatcher } from "./matchers.js";
|
|
3
|
+
import { shouldSuppress } from "./suppressors.js";
|
|
4
|
+
import { detectStructuralDob } from "./structural.js";
|
|
5
|
+
import { COUNTRY_CONFIGS } from "./countries/index.js";
|
|
6
|
+
export class RuleEngine {
|
|
7
|
+
matcher = new MultiPatternMatcher();
|
|
8
|
+
loadedCountries = new Set();
|
|
9
|
+
addCustomPattern(name, pattern) {
|
|
10
|
+
const pdef = {
|
|
11
|
+
entityType: name, pattern, validator: null,
|
|
12
|
+
description: "", contextKeywords: [], requiresContext: false,
|
|
13
|
+
};
|
|
14
|
+
this.matcher.addPattern(pdef, "CUSTOM");
|
|
15
|
+
this.matcher.compile();
|
|
16
|
+
}
|
|
17
|
+
loadCountries(countryCodes) {
|
|
18
|
+
let configs;
|
|
19
|
+
if (countryCodes === null) {
|
|
20
|
+
configs = Object.values(COUNTRY_CONFIGS);
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
configs = [];
|
|
24
|
+
for (const c of countryCodes) {
|
|
25
|
+
const code = c.toUpperCase();
|
|
26
|
+
const config = COUNTRY_CONFIGS[code];
|
|
27
|
+
if (config)
|
|
28
|
+
configs.push(config);
|
|
29
|
+
}
|
|
30
|
+
const shared = COUNTRY_CONFIGS["SHARED"];
|
|
31
|
+
if (shared)
|
|
32
|
+
configs.push(shared);
|
|
33
|
+
}
|
|
34
|
+
let newCountries = false;
|
|
35
|
+
for (const config of configs) {
|
|
36
|
+
if (!this.loadedCountries.has(config.code)) {
|
|
37
|
+
this.matcher.addCountry(config);
|
|
38
|
+
this.loadedCountries.add(config.code);
|
|
39
|
+
newCountries = true;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (newCountries)
|
|
43
|
+
this.matcher.compile();
|
|
44
|
+
}
|
|
45
|
+
detect(text, countryCodes) {
|
|
46
|
+
this.loadCountries(countryCodes);
|
|
47
|
+
let rawMatches = this.matcher.scan(text);
|
|
48
|
+
if (countryCodes !== null) {
|
|
49
|
+
const codesUpper = new Set(countryCodes.map(c => c.toUpperCase()));
|
|
50
|
+
rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED" || m.countryCode === "CUSTOM");
|
|
51
|
+
}
|
|
52
|
+
// Matches that have a validator but fail it create suppression zones:
|
|
53
|
+
// the span is recognisably a specific entity (e.g. IBAN-shaped) so
|
|
54
|
+
// overlapping regex-only matches (license plate, phone) are false
|
|
55
|
+
// positives and must be suppressed.
|
|
56
|
+
const validated = [];
|
|
57
|
+
const suppressionZones = [];
|
|
58
|
+
for (const m of rawMatches) {
|
|
59
|
+
if (this.matcher.validate(m)) {
|
|
60
|
+
validated.push(m);
|
|
61
|
+
}
|
|
62
|
+
else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
|
|
63
|
+
suppressionZones.push([m.start, m.end]);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// Build candidates with priority: validated (3) > custom (2) > regex-only (1)
|
|
67
|
+
const candidates = [];
|
|
68
|
+
for (const match of validated) {
|
|
69
|
+
if (shouldSuppress(text, match))
|
|
70
|
+
continue;
|
|
71
|
+
if (match.patternDef.validator === null && suppressionZones.length > 0) {
|
|
72
|
+
if (suppressionZones.some(([zStart, zEnd]) => match.start >= zStart && match.end <= zEnd)) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const hasValidator = match.patternDef.validator !== null;
|
|
77
|
+
const isValid = this.matcher.validate(match);
|
|
78
|
+
const priority = (hasValidator && isValid) ? 3 : match.countryCode === "CUSTOM" ? 2 : 1;
|
|
79
|
+
const spanLength = match.end - match.start;
|
|
80
|
+
const score = priority * 1_000_000 + spanLength;
|
|
81
|
+
candidates.push({
|
|
82
|
+
det: {
|
|
83
|
+
entityType: match.patternDef.entityType,
|
|
84
|
+
start: match.start,
|
|
85
|
+
end: match.end,
|
|
86
|
+
text: match.text,
|
|
87
|
+
source: DetectionSource.RULES,
|
|
88
|
+
country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM") ? match.countryCode : null,
|
|
89
|
+
confidence: "high",
|
|
90
|
+
},
|
|
91
|
+
score,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
for (const d of detectStructuralDob(text)) {
|
|
95
|
+
candidates.push({ det: d, score: 1_000_000 + (d.end - d.start) });
|
|
96
|
+
}
|
|
97
|
+
return this.deduplicate(candidates);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Remove overlapping detections with priority-aware resolution.
|
|
101
|
+
* Priority: validated (3) > custom (2) > regex-only (1).
|
|
102
|
+
* Within the same tier, longer span wins.
|
|
103
|
+
*/
|
|
104
|
+
deduplicate(candidates) {
|
|
105
|
+
if (candidates.length === 0)
|
|
106
|
+
return [];
|
|
107
|
+
const sorted = [...candidates].sort((a, b) => b.score - a.score);
|
|
108
|
+
const result = [];
|
|
109
|
+
for (const { det } of sorted) {
|
|
110
|
+
let overlaps = false;
|
|
111
|
+
for (const kept of result) {
|
|
112
|
+
if (det.start < kept.end && det.end > kept.start) {
|
|
113
|
+
overlaps = true;
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (!overlaps)
|
|
118
|
+
result.push(det);
|
|
119
|
+
}
|
|
120
|
+
return result;
|
|
121
|
+
}
|
|
122
|
+
getLoadedCountries() {
|
|
123
|
+
return new Set(this.loadedCountries);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { CountryConfig, PatternDef } from "../types.js";
|
|
2
|
+
import type { RawMatch } from "./suppressors.js";
|
|
3
|
+
export type { RawMatch };
|
|
4
|
+
export declare class MultiPatternMatcher {
|
|
5
|
+
private patterns;
|
|
6
|
+
private compiled;
|
|
7
|
+
addPattern(patternDef: PatternDef, countryCode: string): void;
|
|
8
|
+
addCountry(config: CountryConfig): void;
|
|
9
|
+
compile(): void;
|
|
10
|
+
scan(text: string): RawMatch[];
|
|
11
|
+
validate(match: RawMatch): boolean;
|
|
12
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { VALIDATORS } from "./validators.js";
|
|
2
|
+
export class MultiPatternMatcher {
|
|
3
|
+
patterns = [];
|
|
4
|
+
compiled = false;
|
|
5
|
+
addPattern(patternDef, countryCode) {
|
|
6
|
+
const regex = new RegExp(patternDef.pattern, "gu");
|
|
7
|
+
this.patterns.push({ regex, patternDef, countryCode });
|
|
8
|
+
this.compiled = false;
|
|
9
|
+
}
|
|
10
|
+
addCountry(config) {
|
|
11
|
+
for (const pdef of config.patterns) {
|
|
12
|
+
// Convert Python-style \b word boundaries — JS handles them natively
|
|
13
|
+
// But we need the 'u' flag for Unicode support
|
|
14
|
+
const regex = new RegExp(pdef.pattern, "gu");
|
|
15
|
+
this.patterns.push({ regex, patternDef: pdef, countryCode: config.code });
|
|
16
|
+
}
|
|
17
|
+
this.compiled = false;
|
|
18
|
+
}
|
|
19
|
+
compile() {
|
|
20
|
+
this.compiled = true;
|
|
21
|
+
}
|
|
22
|
+
scan(text) {
|
|
23
|
+
if (!this.compiled)
|
|
24
|
+
this.compile();
|
|
25
|
+
const matches = [];
|
|
26
|
+
for (const { regex, patternDef, countryCode } of this.patterns) {
|
|
27
|
+
regex.lastIndex = 0;
|
|
28
|
+
let m;
|
|
29
|
+
while ((m = regex.exec(text)) !== null) {
|
|
30
|
+
matches.push({
|
|
31
|
+
start: m.index,
|
|
32
|
+
end: m.index + m[0].length,
|
|
33
|
+
text: m[0],
|
|
34
|
+
patternDef,
|
|
35
|
+
countryCode,
|
|
36
|
+
});
|
|
37
|
+
// Prevent infinite loops on zero-length matches
|
|
38
|
+
if (m[0].length === 0)
|
|
39
|
+
regex.lastIndex++;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return matches;
|
|
43
|
+
}
|
|
44
|
+
validate(match) {
|
|
45
|
+
if (match.patternDef.validator === null)
|
|
46
|
+
return true;
|
|
47
|
+
const validator = VALIDATORS[match.patternDef.validator];
|
|
48
|
+
if (!validator)
|
|
49
|
+
return true;
|
|
50
|
+
return validator(match.text);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { EntityType, DetectionSource } from "../types.js";
|
|
2
|
+
const DATE_RE = /(?:0[1-9]|[12]\d|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}|(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12]\d|3[01])/g;
|
|
3
|
+
const DOB_FIELD_NAMES = [
|
|
4
|
+
"geboortedatum", "date_naissance", "date de naissance", "geburtsdatum",
|
|
5
|
+
"dob", "birth_date", "birthdate", "date_of_birth",
|
|
6
|
+
"fecha_nacimiento", "data_nascita", "born",
|
|
7
|
+
"naissance", "geboren",
|
|
8
|
+
];
|
|
9
|
+
const JSON_KV = /"([^"]+?)"\s*:\s*"([^"]*?)"/g;
|
|
10
|
+
function detectJsonDob(text) {
|
|
11
|
+
const detections = [];
|
|
12
|
+
JSON_KV.lastIndex = 0;
|
|
13
|
+
let m;
|
|
14
|
+
while ((m = JSON_KV.exec(text)) !== null) {
|
|
15
|
+
const key = m[1].toLowerCase().trim();
|
|
16
|
+
const value = m[2];
|
|
17
|
+
if (!DOB_FIELD_NAMES.some(kw => key.includes(kw)))
|
|
18
|
+
continue;
|
|
19
|
+
DATE_RE.lastIndex = 0;
|
|
20
|
+
const dm = DATE_RE.exec(value);
|
|
21
|
+
if (dm) {
|
|
22
|
+
const valStart = m.index + m[0].indexOf(m[2]);
|
|
23
|
+
const absStart = valStart + dm.index;
|
|
24
|
+
const absEnd = absStart + dm[0].length;
|
|
25
|
+
detections.push({
|
|
26
|
+
entityType: EntityType.DOB,
|
|
27
|
+
start: absStart,
|
|
28
|
+
end: absEnd,
|
|
29
|
+
text: text.slice(absStart, absEnd),
|
|
30
|
+
source: DetectionSource.RULES,
|
|
31
|
+
country: null,
|
|
32
|
+
confidence: "high",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return detections;
|
|
37
|
+
}
|
|
38
|
+
function detectDelimiter(header) {
|
|
39
|
+
for (const delim of ["|", ";", "\t", ","]) {
|
|
40
|
+
if ((header.match(new RegExp(delim.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")) || []).length >= 2)
|
|
41
|
+
return delim;
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
function detectCsvDob(text) {
|
|
46
|
+
const detections = [];
|
|
47
|
+
const lines = text.split("\n");
|
|
48
|
+
if (lines.length < 2)
|
|
49
|
+
return detections;
|
|
50
|
+
const headerLine = lines[0];
|
|
51
|
+
const delimiter = detectDelimiter(headerLine);
|
|
52
|
+
if (!delimiter)
|
|
53
|
+
return detections;
|
|
54
|
+
const headers = headerLine.split(delimiter).map(h => h.trim().toLowerCase());
|
|
55
|
+
if (headers.length < 2)
|
|
56
|
+
return detections;
|
|
57
|
+
const dobColumns = [];
|
|
58
|
+
for (let i = 0; i < headers.length; i++) {
|
|
59
|
+
if (DOB_FIELD_NAMES.some(kw => headers[i].includes(kw)))
|
|
60
|
+
dobColumns.push(i);
|
|
61
|
+
}
|
|
62
|
+
if (dobColumns.length === 0)
|
|
63
|
+
return detections;
|
|
64
|
+
let offset = lines[0].length + 1;
|
|
65
|
+
for (let li = 1; li < lines.length; li++) {
|
|
66
|
+
const line = lines[li];
|
|
67
|
+
if (!line.trim()) {
|
|
68
|
+
offset += line.length + 1;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const fields = line.split(delimiter);
|
|
72
|
+
for (const colIdx of dobColumns) {
|
|
73
|
+
if (colIdx >= fields.length)
|
|
74
|
+
continue;
|
|
75
|
+
const field = fields[colIdx];
|
|
76
|
+
DATE_RE.lastIndex = 0;
|
|
77
|
+
const dm = DATE_RE.exec(field);
|
|
78
|
+
if (dm) {
|
|
79
|
+
let fieldOffset = 0;
|
|
80
|
+
for (let fi = 0; fi < colIdx; fi++)
|
|
81
|
+
fieldOffset += fields[fi].length + delimiter.length;
|
|
82
|
+
const absStart = offset + fieldOffset + dm.index;
|
|
83
|
+
const absEnd = absStart + dm[0].length;
|
|
84
|
+
detections.push({
|
|
85
|
+
entityType: EntityType.DOB,
|
|
86
|
+
start: absStart,
|
|
87
|
+
end: absEnd,
|
|
88
|
+
text: text.slice(absStart, absEnd),
|
|
89
|
+
source: DetectionSource.RULES,
|
|
90
|
+
country: null,
|
|
91
|
+
confidence: "high",
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
offset += line.length + 1;
|
|
96
|
+
}
|
|
97
|
+
return detections;
|
|
98
|
+
}
|
|
99
|
+
export function detectStructuralDob(text) {
|
|
100
|
+
return [...detectJsonDob(text), ...detectCsvDob(text)];
|
|
101
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { EntityType } from "../types.js";
|
|
2
|
+
const CONTEXT_CHARS = 150;
|
|
3
|
+
function getContext(text, start, end) {
|
|
4
|
+
const ctxStart = Math.max(0, start - CONTEXT_CHARS);
|
|
5
|
+
const ctxEnd = Math.min(text.length, end + CONTEXT_CHARS);
|
|
6
|
+
return [text.slice(ctxStart, start), text.slice(end, ctxEnd)];
|
|
7
|
+
}
|
|
8
|
+
// Unicode-safe word boundary: negative lookahead for any letter (including accented)
|
|
9
|
+
const _UWB = "(?![a-zA-Z\\u00C0-\\u024F\\u0400-\\u04FF])";
|
|
10
|
+
const CURRENCY_AFTER = new RegExp(`^\\s*(?:EUR|€|\\$|USD|GBP|£|CHF|ISK|SEK|NOK|DKK|euro|euros|dollar|dollars|pond|kronor|kroner|kr)${_UWB}`, "i");
|
|
11
|
+
const CURRENCY_COMMA_AFTER = new RegExp(`^[.,]\\d{1,2}\\s*(?:EUR|€|\\$|USD|GBP|£|CHF|ISK|SEK|NOK|DKK|euro|euros|kr(?:onor|oner)?|pond)${_UWB}`, "i");
|
|
12
|
+
const CURRENCY_BEFORE = /(?:EUR|€|\$|USD|GBP|£|CHF|ISK|SEK|NOK|DKK)\s*$/;
|
|
13
|
+
const AMOUNT_LABEL_BEFORE = /(?:Montant|Beløb|Summa|Summe|Bedrag|Amount|Total|TTC|inkl|Upphæð)\s*:?\s*$/i;
|
|
14
|
+
// Use Unicode-aware word boundary via \p{L} negative lookahead instead of \b
|
|
15
|
+
// because JS \b is ASCII-only and fails on Unicode letters (e.g. "München" → \bm\b matches M before ü)
|
|
16
|
+
const UNIT_AFTER = /^\s*(?:kg|km|cm|mm|m[²³]|ml|mg|GB|MB|KB|TB|%|jaar|maanden|weken|dagen|uur|minuten|seconden|stuks|pcs|pieces|ans|mois|semaines|jours|heures|Jahre|Monate|Wochen|Tage|Stunden)(?![a-zA-Z\u00C0-\u024F\u0400-\u04FF])/i;
|
|
17
|
+
const REFERENCE_BEFORE = /(?:dossier|ref\.?|referentie|reference|référence|factuurnummer|invoice\s*(?:nr|number|no)?|bestelnummer|order\s*(?:nr|number|no)?|kenmerk|ordernummer|Aktenzeichen|numéro\s*de\s*(?:dossier|facture|commande)|bestellnummer|Rechnungsnummer|artikelnr|article\s*no|contract\s*(?:nr|number|no)?|pagina|page|Seite|blz\.?|Facture\s*n[°o]?|Faktura\s*n[°or]\.?|Lasku\s*n[°or]o?\.?|Rechnung\s*(?:Nr|n[°o])?|faktura\s*(?:nr|n[°o])?|bestilling\s*(?:nr|n[°o])?|bestelling\s*n[°or]\.?|Reikningur\s*nr)\s*[:.]?\s*$/i;
|
|
18
|
+
const LEGAL_BEFORE = /(?:Art(?:ikel|icle|\.)|§|Artikel|Section|Sectie|Afdeling|paragraaf|Absatz|alinéa|punt|point|Punkt|lid)\s*$/i;
|
|
19
|
+
const MATH_BEFORE = /[=+\-×÷*/]\s*$/;
|
|
20
|
+
const MATH_AFTER = /^\s*[=+\-×÷*/]/;
|
|
21
|
+
const SEQUENTIAL_PATTERNS = /^(?:0{6,}|1234567890?|0123456789|9876543210?|1111111111?|000000000|123456789)$/;
|
|
22
|
+
const YEAR_PATTERN = /^(?:19[4-9]\d|20[0-3]\d)$/;
|
|
23
|
+
const DATE_KEYWORD_NEAR = /(?:jaar|year|année|Jahr|datum|date|Datum|in\s+\d{4}|since|sinds|depuis|seit|født|fødselsdato|fødsel|Fødselsdato|född|födelsedatum|födelsedag|syntynyt|syntymäaika|fæddur|fæðingardagur|\d{2}\.\d{2}\.|(?:januar|februar|marts|april|maj|juni|juli|august|september|oktober|november|december|januari|februari|mars|april|mei|juin|juillet|août|Tiltr[æa]delsesdato|Tiltredelsesdato))/i;
|
|
24
|
+
const ID_LABEL_BEFORE = /(?:BSN|RR|NN|NIR|INSZ|NISS|NIS|Steuer-?ID|TIN|NIF|NIE|SSN|rijksregisternummer|numéro\s*national|national\s*number|matricule|Ausweisnummer|Personalausweis|Versichertennummer|KVNR|KV-Nr|Steuernummer|St\.\-Nr|StNr|Finanzamt\s+ist|Ondernemingen\s+onder\s+nummer|ondernemingsnummer|numéro\s*d'entreprise|enterprise\s*number|Kruispuntbank)\s*[:.]?\s*$/i;
|
|
25
|
+
const SERVICE_NUMBER = /^0800[\-\s]/;
|
|
26
|
+
const DATE_PATTERN_FULL = /^\d{2}[-/.]\d{2}[-/.]\d{4}$/;
|
|
27
|
+
const PASSPORT_CONTEXT_BEFORE = /(?:Reisepass|passport|passeport|paspoort|Bisheriger\s+Reisepass)\s*(?:Nr\.?|Nummer|nummer|number|n[°o])?\s*[:.]?\s*$/i;
|
|
28
|
+
const SE_ORG_CONTEXT_BEFORE = /(?:org\.?\s*nr\.?|organisationsnummer|organisationsnr|Bolagsverket|företag)\s*[:.]?\s*$/i;
|
|
29
|
+
const NUMERIC_TYPES = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID, EntityType.POSTAL_CODE]);
|
|
30
|
+
function suppressCurrency(text, match) {
|
|
31
|
+
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
32
|
+
return false;
|
|
33
|
+
const [before, after] = getContext(text, match.start, match.end);
|
|
34
|
+
if (CURRENCY_AFTER.test(after) || CURRENCY_BEFORE.test(before))
|
|
35
|
+
return true;
|
|
36
|
+
if (CURRENCY_COMMA_AFTER.test(after))
|
|
37
|
+
return true;
|
|
38
|
+
if (AMOUNT_LABEL_BEFORE.test(before))
|
|
39
|
+
return true;
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
function suppressUnits(text, match) {
|
|
43
|
+
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
44
|
+
return false;
|
|
45
|
+
const [, after] = getContext(text, match.start, match.end);
|
|
46
|
+
return UNIT_AFTER.test(after);
|
|
47
|
+
}
|
|
48
|
+
function suppressReference(text, match) {
|
|
49
|
+
const applicable = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.SSN, EntityType.TAX_ID, EntityType.IBAN, EntityType.CHAMBER_OF_COMMERCE]);
|
|
50
|
+
if (!applicable.has(match.patternDef.entityType))
|
|
51
|
+
return false;
|
|
52
|
+
const [before] = getContext(text, match.start, match.end);
|
|
53
|
+
return REFERENCE_BEFORE.test(before);
|
|
54
|
+
}
|
|
55
|
+
function suppressLegal(text, match) {
|
|
56
|
+
const applicable = new Set([EntityType.PHONE, EntityType.NATIONAL_ID, EntityType.POSTAL_CODE]);
|
|
57
|
+
if (!applicable.has(match.patternDef.entityType))
|
|
58
|
+
return false;
|
|
59
|
+
const [before] = getContext(text, match.start, match.end);
|
|
60
|
+
return LEGAL_BEFORE.test(before);
|
|
61
|
+
}
|
|
62
|
+
function suppressMath(text, match) {
|
|
63
|
+
if (!NUMERIC_TYPES.has(match.patternDef.entityType))
|
|
64
|
+
return false;
|
|
65
|
+
const [before, after] = getContext(text, match.start, match.end);
|
|
66
|
+
return MATH_BEFORE.test(before) || MATH_AFTER.test(after);
|
|
67
|
+
}
|
|
68
|
+
function suppressSequential(_text, match) {
|
|
69
|
+
const clean = match.text.replace(/[\s.\-]/g, "");
|
|
70
|
+
return SEQUENTIAL_PATTERNS.test(clean);
|
|
71
|
+
}
|
|
72
|
+
function suppressYearAsPostal(text, match) {
|
|
73
|
+
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
74
|
+
return false;
|
|
75
|
+
const clean = match.text.trim();
|
|
76
|
+
if (!YEAR_PATTERN.test(clean))
|
|
77
|
+
return false;
|
|
78
|
+
const immediateBefore = text.slice(Math.max(0, match.start - 3), match.start);
|
|
79
|
+
if (immediateBefore.endsWith(", ") || immediateBefore.endsWith(",\n"))
|
|
80
|
+
return false;
|
|
81
|
+
const [before, after] = getContext(text, match.start, match.end);
|
|
82
|
+
return DATE_KEYWORD_NEAR.test(before + after);
|
|
83
|
+
}
|
|
84
|
+
function suppressPhoneAfterIdLabel(text, match) {
|
|
85
|
+
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
86
|
+
return false;
|
|
87
|
+
const [before] = getContext(text, match.start, match.end);
|
|
88
|
+
return ID_LABEL_BEFORE.test(before);
|
|
89
|
+
}
|
|
90
|
+
function suppressPhoneServiceNumber(_text, match) {
|
|
91
|
+
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
92
|
+
return false;
|
|
93
|
+
return SERVICE_NUMBER.test(match.text);
|
|
94
|
+
}
|
|
95
|
+
function suppressPhoneDateOverlap(_text, match) {
|
|
96
|
+
if (match.patternDef.entityType !== EntityType.PHONE)
|
|
97
|
+
return false;
|
|
98
|
+
return DATE_PATTERN_FULL.test(match.text.trim());
|
|
99
|
+
}
|
|
100
|
+
const NOT_CITY_CODES = new Set([
|
|
101
|
+
"ID", "NR", "NO", "ST", "DR", "MR", "MS", "HR", "FR", "IM", "IN", "OR", "IF", "IS", "IT", "AT", "AD", "AG", "AV",
|
|
102
|
+
"BE", "DE", "EU", "NL", "LU", "WS", "SS", "IP",
|
|
103
|
+
]);
|
|
104
|
+
function suppressPlateInCompound(text, match) {
|
|
105
|
+
if (match.patternDef.entityType !== EntityType.LICENSE_PLATE)
|
|
106
|
+
return false;
|
|
107
|
+
if (match.start >= 3) {
|
|
108
|
+
const before = text.slice(Math.max(0, match.start - 10), match.start);
|
|
109
|
+
if (/[A-Za-zÄÖÜäöüß]{2,}-$/.test(before))
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
const matched = match.text.trim();
|
|
113
|
+
const parts = matched.split(/[\s\-]+/);
|
|
114
|
+
if (parts.length > 0 && NOT_CITY_CODES.has(parts[0])) {
|
|
115
|
+
const afterChar = match.end < text.length ? text[match.end] : "";
|
|
116
|
+
const beforeChar = match.start > 0 ? text[match.start - 1] : "";
|
|
117
|
+
if (/\d/.test(afterChar) || beforeChar === "-")
|
|
118
|
+
return true;
|
|
119
|
+
if (parts[0] === "WS" || parts[0] === "SS")
|
|
120
|
+
return true;
|
|
121
|
+
if (parts[0] === "IP") {
|
|
122
|
+
const afterTwo = text.slice(match.end, match.end + 2);
|
|
123
|
+
if (afterTwo.length >= 2 && afterTwo[0] === "." && /\d/.test(afterTwo[1]))
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (matched.startsWith("HRA") || matched.startsWith("HRB"))
|
|
128
|
+
return true;
|
|
129
|
+
const [before, after] = getContext(text, match.start, match.end);
|
|
130
|
+
if (/(?:Semester|Hochschule|Uni\b)/i.test(before + after)) {
|
|
131
|
+
if (parts.length > 0 && (parts[0] === "WS" || parts[0] === "SS"))
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
function suppressNatidAsPassport(text, match) {
|
|
137
|
+
if (match.patternDef.entityType !== EntityType.NATIONAL_ID)
|
|
138
|
+
return false;
|
|
139
|
+
const [before] = getContext(text, match.start, match.end);
|
|
140
|
+
return PASSPORT_CONTEXT_BEFORE.test(before);
|
|
141
|
+
}
|
|
142
|
+
function suppressSeNatidAsOrg(text, match) {
|
|
143
|
+
if (match.patternDef.entityType !== EntityType.NATIONAL_ID)
|
|
144
|
+
return false;
|
|
145
|
+
if (match.countryCode !== "SE")
|
|
146
|
+
return false;
|
|
147
|
+
const [before] = getContext(text, match.start, match.end);
|
|
148
|
+
return SE_ORG_CONTEXT_BEFORE.test(before);
|
|
149
|
+
}
|
|
150
|
+
function suppressPostalInsideIban(text, match) {
|
|
151
|
+
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
152
|
+
return false;
|
|
153
|
+
const beforeChar = match.start > 0 ? text[match.start - 1] : " ";
|
|
154
|
+
const afterChar = match.end < text.length ? text[match.end] : " ";
|
|
155
|
+
if (/[A-Za-z0-9]/.test(beforeChar) && /[A-Za-z0-9]/.test(afterChar))
|
|
156
|
+
return true;
|
|
157
|
+
if (match.start >= 5) {
|
|
158
|
+
const prefix = text.slice(match.start - 5, match.start);
|
|
159
|
+
if (/[A-Z]{2}\d{2}\s$/.test(prefix))
|
|
160
|
+
return true;
|
|
161
|
+
}
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
function suppressPostalAsHouseNumber(text, match) {
|
|
165
|
+
if (match.patternDef.entityType !== EntityType.POSTAL_CODE)
|
|
166
|
+
return false;
|
|
167
|
+
const clean = match.text.replace(/\s/g, "");
|
|
168
|
+
if (clean.length > 3)
|
|
169
|
+
return false;
|
|
170
|
+
const before = text.slice(Math.max(0, match.start - 30), match.start);
|
|
171
|
+
if (/[a-záéíóúýþæöðA-ZÁÉÍÓÚÝÞÆÖÐ]{3,}\s+$/.test(before)) {
|
|
172
|
+
const after = text.slice(match.end, match.end + 5);
|
|
173
|
+
if (!/^,?\s+[A-ZÁÉÍÓÚÝÞÆÖÐ]/.test(after))
|
|
174
|
+
return true;
|
|
175
|
+
}
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
function suppressRequiresContext(text, match) {
|
|
179
|
+
if (!match.patternDef.requiresContext || match.patternDef.contextKeywords.length === 0)
|
|
180
|
+
return false;
|
|
181
|
+
const [before, after] = getContext(text, match.start, match.end);
|
|
182
|
+
const context = (before + " " + after).toLowerCase();
|
|
183
|
+
return !match.patternDef.contextKeywords.some(kw => context.includes(kw.toLowerCase()));
|
|
184
|
+
}
|
|
185
|
+
const TYPE_SUPPRESSORS = {
|
|
186
|
+
[EntityType.PHONE]: [suppressCurrency, suppressUnits, suppressReference, suppressMath, suppressPhoneAfterIdLabel, suppressPhoneServiceNumber, suppressPhoneDateOverlap],
|
|
187
|
+
[EntityType.NATIONAL_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressLegal, suppressMath, suppressNatidAsPassport, suppressSeNatidAsOrg],
|
|
188
|
+
[EntityType.SSN]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
189
|
+
[EntityType.TAX_ID]: [suppressCurrency, suppressUnits, suppressReference, suppressMath],
|
|
190
|
+
[EntityType.POSTAL_CODE]: [suppressCurrency, suppressUnits, suppressMath, suppressLegal, suppressYearAsPostal, suppressPostalInsideIban, suppressPostalAsHouseNumber],
|
|
191
|
+
[EntityType.IBAN]: [suppressReference],
|
|
192
|
+
[EntityType.LICENSE_PLATE]: [suppressPlateInCompound],
|
|
193
|
+
[EntityType.CHAMBER_OF_COMMERCE]: [suppressReference],
|
|
194
|
+
};
|
|
195
|
+
export function shouldSuppress(text, match) {
|
|
196
|
+
if (suppressSequential(text, match))
|
|
197
|
+
return true;
|
|
198
|
+
const typeSups = TYPE_SUPPRESSORS[match.patternDef.entityType];
|
|
199
|
+
if (typeSups) {
|
|
200
|
+
for (const s of typeSups) {
|
|
201
|
+
if (s(text, match))
|
|
202
|
+
return true;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (match.patternDef.requiresContext)
|
|
206
|
+
return suppressRequiresContext(text, match);
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export declare function validateIban(candidate: string): boolean;
|
|
2
|
+
export declare function validateBsn(candidate: string): boolean;
|
|
3
|
+
export declare function validateBelgianNn(candidate: string): boolean;
|
|
4
|
+
export declare function validateLuhn(candidate: string): boolean;
|
|
5
|
+
export declare function validateBelgianVat(candidate: string): boolean;
|
|
6
|
+
export declare function validateVatNl(candidate: string): boolean;
|
|
7
|
+
export declare function validateVatDe(candidate: string): boolean;
|
|
8
|
+
export declare function validateVatFr(candidate: string): boolean;
|
|
9
|
+
export declare function validateVatLu(candidate: string): boolean;
|
|
10
|
+
export declare function validateGermanTaxId(candidate: string): boolean;
|
|
11
|
+
export declare function validateFrenchNir(candidate: string): boolean;
|
|
12
|
+
export declare function validateVin(candidate: string): boolean;
|
|
13
|
+
export declare function validateBic(candidate: string): boolean;
|
|
14
|
+
export declare function validateKvk(candidate: string): boolean;
|
|
15
|
+
export declare function validateSwedishPnr(candidate: string): boolean;
|
|
16
|
+
export declare function validateNorwegianFnr(candidate: string): boolean;
|
|
17
|
+
export declare function validateFinnishHetu(candidate: string): boolean;
|
|
18
|
+
export declare function validateIcelandicKt(candidate: string): boolean;
|
|
19
|
+
export declare function validateDanishVat(candidate: string): boolean;
|
|
20
|
+
export declare function validateFinnishBusinessId(candidate: string): boolean;
|
|
21
|
+
export declare function validateImei(candidate: string): boolean;
|
|
22
|
+
export declare function validateDanishCpr(candidate: string): boolean;
|
|
23
|
+
export declare function validateNorwegianOrg(candidate: string): boolean;
|
|
24
|
+
export declare function validateAustrianSvnr(candidate: string): boolean;
|
|
25
|
+
export declare function validateSwissAhv(candidate: string): boolean;
|
|
26
|
+
export declare function validateItalianCf(candidate: string): boolean;
|
|
27
|
+
export declare function validateSpanishDni(candidate: string): boolean;
|
|
28
|
+
export declare function validateSpanishNie(candidate: string): boolean;
|
|
29
|
+
export declare function validatePortugueseNif(candidate: string): boolean;
|
|
30
|
+
export declare function validatePolishPesel(candidate: string): boolean;
|
|
31
|
+
export declare function validatePolishNip(candidate: string): boolean;
|
|
32
|
+
export declare function validateCzechBirthNumber(candidate: string): boolean;
|
|
33
|
+
export declare function validateRomanianCnp(candidate: string): boolean;
|
|
34
|
+
export declare function validateHungarianTaj(candidate: string): boolean;
|
|
35
|
+
export declare function validateBulgarianEgn(candidate: string): boolean;
|
|
36
|
+
export declare function validateCroatianOib(candidate: string): boolean;
|
|
37
|
+
export declare function validateSlovenianEmso(candidate: string): boolean;
|
|
38
|
+
export declare function validateIrishPps(candidate: string): boolean;
|
|
39
|
+
export declare function validateEstonianId(candidate: string): boolean;
|
|
40
|
+
export declare function validateUkNhs(candidate: string): boolean;
|
|
41
|
+
export declare function validateGreekAfm(candidate: string): boolean;
|
|
42
|
+
export declare function validateGreekAmka(candidate: string): boolean;
|
|
43
|
+
export declare const VALIDATORS: Record<string, (candidate: string) => boolean>;
|