@stll/anonymize-wasm 1.4.4 → 1.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-boundaries.mjs +2 -0
- package/dist/constants.mjs +3 -0
- package/dist/constants.mjs.map +1 -1
- package/dist/constants2.d.mts +3 -1
- package/dist/document-structure-headings.mjs +90 -0
- package/dist/document-structure-headings.mjs.map +1 -0
- package/dist/names-exclusions.mjs +12 -0
- package/dist/person-stopwords.mjs +10 -0
- package/dist/sentence-verb-indicators.mjs +10 -1
- package/dist/triggers.de.mjs +2 -2
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +39 -17
- package/dist/wasm.mjs +7546 -229
- package/dist/wasm.mjs.map +1 -1
- package/package.json +7 -3
- package/dist/address-street-types.mjs +0 -244
- package/dist/address-street-types.mjs.map +0 -1
- package/dist/amount-words.mjs +0 -478
- package/dist/amount-words.mjs.map +0 -1
- package/dist/rolldown-runtime.mjs +0 -13
package/dist/constants.mjs
CHANGED
|
@@ -25,6 +25,7 @@ const DETECTION_SOURCES = {
|
|
|
25
25
|
DENY_LIST: "deny-list",
|
|
26
26
|
LEGAL_FORM: "legal-form",
|
|
27
27
|
GAZETTEER: "gazetteer",
|
|
28
|
+
COUNTRY: "country",
|
|
28
29
|
NER: "ner",
|
|
29
30
|
COREFERENCE: "coreference"
|
|
30
31
|
};
|
|
@@ -39,6 +40,7 @@ const DETECTOR_PRIORITY = {
|
|
|
39
40
|
[DETECTION_SOURCES.TRIGGER]: 4,
|
|
40
41
|
[DETECTION_SOURCES.LEGAL_FORM]: 3,
|
|
41
42
|
[DETECTION_SOURCES.REGEX]: 3,
|
|
43
|
+
[DETECTION_SOURCES.COUNTRY]: 3,
|
|
42
44
|
[DETECTION_SOURCES.DENY_LIST]: 2,
|
|
43
45
|
[DETECTION_SOURCES.COREFERENCE]: 2,
|
|
44
46
|
[DETECTION_SOURCES.NER]: 1
|
|
@@ -62,6 +64,7 @@ const DEFAULT_ENTITY_LABELS = [
|
|
|
62
64
|
"organization",
|
|
63
65
|
"phone number",
|
|
64
66
|
"address",
|
|
67
|
+
"country",
|
|
65
68
|
"email address",
|
|
66
69
|
"date",
|
|
67
70
|
"date of birth",
|
package/dist/constants.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.mjs","names":[],"sources":["../../src/constants.ts"],"sourcesContent":["/**\n * Runtime-free constants for the anonymization pipeline.\n *\n * This module is the SSR-safe / browser-safe entrypoint:\n * importing it must not pull in `@stll/text-search`,\n * `@stll/anonymize-wasm`, or any other runtime-bearing\n * module. Consumers that only need the static label list,\n * detection-source identifiers, or operator names import\n * from `@stll/anonymize/constants` (or\n * `@stll/anonymize-wasm/constants`) without paying the\n * wasm / regex-set startup cost.\n *\n * `types.ts` re-exports these for back-compat, so existing\n * `import { DEFAULT_ENTITY_LABELS } from \"@stll/anonymize\"`\n * call sites keep working.\n */\n\n/**\n * Source of a detected entity span.\n * Ordered by detection layer in the pipeline.\n */\nexport const DETECTION_SOURCES = {\n TRIGGER: \"trigger\",\n REGEX: \"regex\",\n DENY_LIST: \"deny-list\",\n LEGAL_FORM: \"legal-form\",\n GAZETTEER: \"gazetteer\",\n NER: \"ner\",\n COREFERENCE: \"coreference\",\n} as const;\n\nexport type DetectionSource =\n (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];\n\n/**\n * Priority levels for detection sources.\n * Higher = more structurally reliable. Used during\n * overlap resolution so deterministic detectors beat\n * probabilistic ones regardless of raw score.\n */\nexport const DETECTOR_PRIORITY = {\n [DETECTION_SOURCES.GAZETTEER]: 5,\n [DETECTION_SOURCES.TRIGGER]: 4,\n [DETECTION_SOURCES.LEGAL_FORM]: 3,\n [DETECTION_SOURCES.REGEX]: 3,\n [DETECTION_SOURCES.DENY_LIST]: 2,\n [DETECTION_SOURCES.COREFERENCE]: 2,\n [DETECTION_SOURCES.NER]: 1,\n} as const satisfies Record<DetectionSource, number>;\n\n/**\n * Anonymization operator types. Each operator defines\n * how a confirmed entity is replaced in the output.\n */\nexport const OPERATOR_TYPES = [\"replace\", \"redact\"] as const;\n\nexport type OperatorType = (typeof OPERATOR_TYPES)[number];\n\n/**\n * Canonical entity labels used across the pipeline.\n * NER models may use different native labels; the bench\n * NER wrapper maps model output to these canonical names.\n *\n * These labels are ephemeral: entities are regenerated on\n * every pipeline run and never persisted to the database.\n * Renaming a label here requires no migration.\n */\nexport const DEFAULT_ENTITY_LABELS = [\n \"person\",\n \"organization\",\n \"phone number\",\n \"address\",\n \"email address\",\n \"date\",\n \"date of birth\",\n \"bank account number\",\n \"iban\",\n \"tax identification number\",\n \"identity card number\",\n \"birth number\",\n \"national identification number\",\n \"social security number\",\n \"registration number\",\n \"credit card number\",\n \"passport number\",\n \"monetary amount\",\n \"land parcel\",\n \"misc\",\n] as const;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAqBA,MAAa,oBAAoB;CAC/B,SAAS;CACT,OAAO;CACP,WAAW;CACX,YAAY;CACZ,WAAW;CACX,KAAK;CACL,aAAa;
|
|
1
|
+
{"version":3,"file":"constants.mjs","names":[],"sources":["../../src/constants.ts"],"sourcesContent":["/**\n * Runtime-free constants for the anonymization pipeline.\n *\n * This module is the SSR-safe / browser-safe entrypoint:\n * importing it must not pull in `@stll/text-search`,\n * `@stll/anonymize-wasm`, or any other runtime-bearing\n * module. Consumers that only need the static label list,\n * detection-source identifiers, or operator names import\n * from `@stll/anonymize/constants` (or\n * `@stll/anonymize-wasm/constants`) without paying the\n * wasm / regex-set startup cost.\n *\n * `types.ts` re-exports these for back-compat, so existing\n * `import { DEFAULT_ENTITY_LABELS } from \"@stll/anonymize\"`\n * call sites keep working.\n */\n\n/**\n * Source of a detected entity span.\n * Ordered by detection layer in the pipeline.\n */\nexport const DETECTION_SOURCES = {\n TRIGGER: \"trigger\",\n REGEX: \"regex\",\n DENY_LIST: \"deny-list\",\n LEGAL_FORM: \"legal-form\",\n GAZETTEER: \"gazetteer\",\n COUNTRY: \"country\",\n NER: \"ner\",\n COREFERENCE: \"coreference\",\n} as const;\n\nexport type DetectionSource =\n (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];\n\n/**\n * Priority levels for detection sources.\n * Higher = more structurally reliable. Used during\n * overlap resolution so deterministic detectors beat\n * probabilistic ones regardless of raw score.\n */\nexport const DETECTOR_PRIORITY = {\n [DETECTION_SOURCES.GAZETTEER]: 5,\n [DETECTION_SOURCES.TRIGGER]: 4,\n [DETECTION_SOURCES.LEGAL_FORM]: 3,\n [DETECTION_SOURCES.REGEX]: 3,\n [DETECTION_SOURCES.COUNTRY]: 3,\n [DETECTION_SOURCES.DENY_LIST]: 2,\n [DETECTION_SOURCES.COREFERENCE]: 2,\n [DETECTION_SOURCES.NER]: 1,\n} as const satisfies Record<DetectionSource, number>;\n\n/**\n * Anonymization operator types. Each operator defines\n * how a confirmed entity is replaced in the output.\n */\nexport const OPERATOR_TYPES = [\"replace\", \"redact\"] as const;\n\nexport type OperatorType = (typeof OPERATOR_TYPES)[number];\n\n/**\n * Canonical entity labels used across the pipeline.\n * NER models may use different native labels; the bench\n * NER wrapper maps model output to these canonical names.\n *\n * These labels are ephemeral: entities are regenerated on\n * every pipeline run and never persisted to the database.\n * Renaming a label here requires no migration.\n */\nexport const DEFAULT_ENTITY_LABELS = [\n \"person\",\n \"organization\",\n \"phone number\",\n \"address\",\n \"country\",\n \"email address\",\n \"date\",\n \"date of birth\",\n \"bank account number\",\n \"iban\",\n \"tax identification number\",\n \"identity card number\",\n \"birth number\",\n \"national identification number\",\n \"social security number\",\n \"registration number\",\n \"credit card number\",\n \"passport number\",\n \"monetary amount\",\n \"land parcel\",\n \"misc\",\n] as const;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAqBA,MAAa,oBAAoB;CAC/B,SAAS;CACT,OAAO;CACP,WAAW;CACX,YAAY;CACZ,WAAW;CACX,SAAS;CACT,KAAK;CACL,aAAa;AACf;;;;;;;AAWA,MAAa,oBAAoB;EAC9B,kBAAkB,YAAY;EAC9B,kBAAkB,UAAU;EAC5B,kBAAkB,aAAa;EAC/B,kBAAkB,QAAQ;EAC1B,kBAAkB,UAAU;EAC5B,kBAAkB,YAAY;EAC9B,kBAAkB,cAAc;EAChC,kBAAkB,MAAM;AAC3B;;;;;AAMA,MAAa,iBAAiB,CAAC,WAAW,QAAQ;;;;;;;;;;AAalD,MAAa,wBAAwB;CACnC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}
|
package/dist/constants2.d.mts
CHANGED
|
@@ -25,6 +25,7 @@ declare const DETECTION_SOURCES: {
|
|
|
25
25
|
readonly DENY_LIST: "deny-list";
|
|
26
26
|
readonly LEGAL_FORM: "legal-form";
|
|
27
27
|
readonly GAZETTEER: "gazetteer";
|
|
28
|
+
readonly COUNTRY: "country";
|
|
28
29
|
readonly NER: "ner";
|
|
29
30
|
readonly COREFERENCE: "coreference";
|
|
30
31
|
};
|
|
@@ -40,6 +41,7 @@ declare const DETECTOR_PRIORITY: {
|
|
|
40
41
|
readonly trigger: 4;
|
|
41
42
|
readonly "legal-form": 3;
|
|
42
43
|
readonly regex: 3;
|
|
44
|
+
readonly country: 3;
|
|
43
45
|
readonly "deny-list": 2;
|
|
44
46
|
readonly coreference: 2;
|
|
45
47
|
readonly ner: 1;
|
|
@@ -59,7 +61,7 @@ type OperatorType = (typeof OPERATOR_TYPES)[number];
|
|
|
59
61
|
* every pipeline run and never persisted to the database.
|
|
60
62
|
* Renaming a label here requires no migration.
|
|
61
63
|
*/
|
|
62
|
-
declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "birth number", "national identification number", "social security number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel", "misc"];
|
|
64
|
+
declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "country", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "birth number", "national identification number", "social security number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel", "misc"];
|
|
63
65
|
//#endregion
|
|
64
66
|
export { OPERATOR_TYPES as a, DetectionSource as i, DETECTION_SOURCES as n, OperatorType as o, DETECTOR_PRIORITY as r, DEFAULT_ENTITY_LABELS as t };
|
|
65
67
|
//# sourceMappingURL=constants2.d.mts.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
//#region src/data/document-structure-headings.json
|
|
2
|
+
var _comment = "Document-structure heading words per language. Captured by the trigger detector as organizations when followed by 'no. N' / 'č. N' / 'Nr. N' / similar, they are scaffolding (appendix titles, schedule headings, exhibit labels), not party names. Each entry lists the heading words; the false-positive filter rejects an organization span whose first word matches one of these followed by a numerical ordinal abbreviation and a digit. Lowercased; matched case-insensitively. Organized per-language because the heading vocabulary is language-specific; at runtime the filter unions every language's list into a single set because match spans don't carry a language tag.";
|
|
3
|
+
var cs = [
|
|
4
|
+
"příloha",
|
|
5
|
+
"dodatek",
|
|
6
|
+
"schéma",
|
|
7
|
+
"exhibit",
|
|
8
|
+
"tabulka"
|
|
9
|
+
];
|
|
10
|
+
var sk = [
|
|
11
|
+
"príloha",
|
|
12
|
+
"dodatok",
|
|
13
|
+
"exhibit",
|
|
14
|
+
"tabuľka"
|
|
15
|
+
];
|
|
16
|
+
var pl = [
|
|
17
|
+
"załącznik",
|
|
18
|
+
"aneks",
|
|
19
|
+
"schemat",
|
|
20
|
+
"tabela"
|
|
21
|
+
];
|
|
22
|
+
var de = [
|
|
23
|
+
"anlage",
|
|
24
|
+
"anhang",
|
|
25
|
+
"anlage",
|
|
26
|
+
"exhibit",
|
|
27
|
+
"tabelle"
|
|
28
|
+
];
|
|
29
|
+
var en = [
|
|
30
|
+
"appendix",
|
|
31
|
+
"annex",
|
|
32
|
+
"exhibit",
|
|
33
|
+
"schedule",
|
|
34
|
+
"attachment",
|
|
35
|
+
"table"
|
|
36
|
+
];
|
|
37
|
+
var fr = [
|
|
38
|
+
"annexe",
|
|
39
|
+
"appendice",
|
|
40
|
+
"tableau",
|
|
41
|
+
"exhibit"
|
|
42
|
+
];
|
|
43
|
+
var it = [
|
|
44
|
+
"allegato",
|
|
45
|
+
"appendice",
|
|
46
|
+
"tabella",
|
|
47
|
+
"exhibit"
|
|
48
|
+
];
|
|
49
|
+
var es = [
|
|
50
|
+
"anexo",
|
|
51
|
+
"apéndice",
|
|
52
|
+
"tabla",
|
|
53
|
+
"exhibit"
|
|
54
|
+
];
|
|
55
|
+
var pt_br = [
|
|
56
|
+
"anexo",
|
|
57
|
+
"apêndice",
|
|
58
|
+
"tabela",
|
|
59
|
+
"exhibit"
|
|
60
|
+
];
|
|
61
|
+
var ro = [
|
|
62
|
+
"anexa",
|
|
63
|
+
"apendice",
|
|
64
|
+
"tabel",
|
|
65
|
+
"exhibit"
|
|
66
|
+
];
|
|
67
|
+
var hu = [
|
|
68
|
+
"melléklet",
|
|
69
|
+
"függelék",
|
|
70
|
+
"táblázat",
|
|
71
|
+
"exhibit"
|
|
72
|
+
];
|
|
73
|
+
var document_structure_headings_default = {
|
|
74
|
+
_comment,
|
|
75
|
+
cs,
|
|
76
|
+
sk,
|
|
77
|
+
pl,
|
|
78
|
+
de,
|
|
79
|
+
en,
|
|
80
|
+
fr,
|
|
81
|
+
it,
|
|
82
|
+
es,
|
|
83
|
+
"pt-br": pt_br,
|
|
84
|
+
ro,
|
|
85
|
+
hu
|
|
86
|
+
};
|
|
87
|
+
//#endregion
|
|
88
|
+
export { _comment, cs, de, document_structure_headings_default as default, en, es, fr, hu, it, pl, pt_br as "pt-br", ro, sk };
|
|
89
|
+
|
|
90
|
+
//# sourceMappingURL=document-structure-headings.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-structure-headings.mjs","names":[],"sources":["../../src/data/document-structure-headings.json"],"sourcesContent":[""],"mappings":""}
|
|
@@ -144,16 +144,28 @@ var words = [
|
|
|
144
144
|
"subject",
|
|
145
145
|
"supplier",
|
|
146
146
|
"tarif",
|
|
147
|
+
"tato",
|
|
148
|
+
"tedy",
|
|
149
|
+
"tehdy",
|
|
147
150
|
"tel",
|
|
148
151
|
"telefon",
|
|
149
152
|
"tenant",
|
|
153
|
+
"tento",
|
|
150
154
|
"termín",
|
|
151
155
|
"that",
|
|
152
156
|
"the",
|
|
153
157
|
"therefore",
|
|
154
158
|
"thereof",
|
|
155
159
|
"this",
|
|
160
|
+
"tito",
|
|
161
|
+
"tomto",
|
|
162
|
+
"tomuto",
|
|
163
|
+
"toto",
|
|
164
|
+
"touto",
|
|
156
165
|
"tribunal",
|
|
166
|
+
"tuto",
|
|
167
|
+
"tyto",
|
|
168
|
+
"této",
|
|
157
169
|
"under",
|
|
158
170
|
"until",
|
|
159
171
|
"upon",
|
|
@@ -140,6 +140,16 @@ var person_stopwords_default = {
|
|
|
140
140
|
"viale",
|
|
141
141
|
"vico",
|
|
142
142
|
"vicolo",
|
|
143
|
+
"tato",
|
|
144
|
+
"tento",
|
|
145
|
+
"tito",
|
|
146
|
+
"tomto",
|
|
147
|
+
"tomuto",
|
|
148
|
+
"toto",
|
|
149
|
+
"touto",
|
|
150
|
+
"tuto",
|
|
151
|
+
"tyto",
|
|
152
|
+
"této",
|
|
143
153
|
"acheteur",
|
|
144
154
|
"acheteuse",
|
|
145
155
|
"acquéreur",
|
package/dist/triggers.de.mjs
CHANGED
|
@@ -49,8 +49,8 @@ var triggers_de_default = [
|
|
|
49
49
|
"id": "de-iban",
|
|
50
50
|
"label": "iban",
|
|
51
51
|
"strategy": {
|
|
52
|
-
"type": "
|
|
53
|
-
"
|
|
52
|
+
"type": "match-pattern",
|
|
53
|
+
"pattern": "\\b[A-Z]{2}\\d{2}(?=(?:\\s?[\\dA-Z]){11,30}\\b)(?:\\s?[\\dA-Z]{1,4}){3,8}\\b"
|
|
54
54
|
},
|
|
55
55
|
"triggers": ["IBAN"],
|
|
56
56
|
"extensions": ["add-colon"]
|
package/dist/vite.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vite.mjs","names":[],"sources":["../../src/vite.ts"],"sourcesContent":["/* Vite plugin that wires up @stll/anonymize-wasm so its transitive\n * napi-rs-generated wasm loaders survive Vite's dep pre-bundler. Adds\n * this package plus its @stll/*-wasm dependencies to optimizeDeps.exclude\n * so the loader modules are served with their original paths, keeping\n * `new URL(\"./foo.wasm\", import.meta.url)` correct. */\nimport type { Plugin } from \"vite\";\n\n// Only the napi-rs WASM packages need excluding — they use\n// `new URL(\"./foo.wasm\", import.meta.url)` which Vite's optimizer\n// would rewrite. @stll/anonymize-wasm itself is pure JS and should\n// NOT be excluded.\nconst PACKAGES = [\n \"@stll/text-search-wasm\",\n \"@stll/aho-corasick-wasm\",\n \"@stll/aho-corasick-wasm32-wasi\",\n \"@stll/fuzzy-search-wasm\",\n \"@stll/fuzzy-search-wasm32-wasi\",\n \"@stll/regex-set-wasm\",\n \"@stll/regex-set-wasm32-wasi\",\n];\n\nexport default function stllAnonymizeWasmVite(): Plugin {\n return {\n name: \"stll-anonymize-wasm\",\n apply: \"serve\",\n config() {\n return {\n optimizeDeps: { exclude: PACKAGES },\n };\n },\n };\n}\n"],"mappings":";AAWA,MAAM,WAAW;CACf;CACA;CACA;CACA;CACA;CACA;CACA;
|
|
1
|
+
{"version":3,"file":"vite.mjs","names":[],"sources":["../../src/vite.ts"],"sourcesContent":["/* Vite plugin that wires up @stll/anonymize-wasm so its transitive\n * napi-rs-generated wasm loaders survive Vite's dep pre-bundler. Adds\n * this package plus its @stll/*-wasm dependencies to optimizeDeps.exclude\n * so the loader modules are served with their original paths, keeping\n * `new URL(\"./foo.wasm\", import.meta.url)` correct. */\nimport type { Plugin } from \"vite\";\n\n// Only the napi-rs WASM packages need excluding — they use\n// `new URL(\"./foo.wasm\", import.meta.url)` which Vite's optimizer\n// would rewrite. @stll/anonymize-wasm itself is pure JS and should\n// NOT be excluded.\nconst PACKAGES = [\n \"@stll/text-search-wasm\",\n \"@stll/aho-corasick-wasm\",\n \"@stll/aho-corasick-wasm32-wasi\",\n \"@stll/fuzzy-search-wasm\",\n \"@stll/fuzzy-search-wasm32-wasi\",\n \"@stll/regex-set-wasm\",\n \"@stll/regex-set-wasm32-wasi\",\n];\n\nexport default function stllAnonymizeWasmVite(): Plugin {\n return {\n name: \"stll-anonymize-wasm\",\n apply: \"serve\",\n config() {\n return {\n optimizeDeps: { exclude: PACKAGES },\n };\n },\n };\n}\n"],"mappings":";AAWA,MAAM,WAAW;CACf;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAwB,wBAAgC;CACtD,OAAO;EACL,MAAM;EACN,OAAO;EACP,SAAS;GACP,OAAO,EACL,cAAc,EAAE,SAAS,SAAS,EACpC;EACF;CACF;AACF"}
|
package/dist/wasm.d.mts
CHANGED
|
@@ -325,6 +325,13 @@ type PipelineConfig = {
|
|
|
325
325
|
*/
|
|
326
326
|
customRegexes?: readonly CustomRegexPattern[];
|
|
327
327
|
enableGazetteer: boolean;
|
|
328
|
+
/**
|
|
329
|
+
* Detect country names (ISO 3166-1 names, curated
|
|
330
|
+
* aliases, alpha-3 codes). Defaults to true. Names
|
|
331
|
+
* span all manifest languages plus widely-used
|
|
332
|
+
* additions (Dutch, Russian, Chinese, Arabic, etc.).
|
|
333
|
+
*/
|
|
334
|
+
enableCountries?: boolean;
|
|
328
335
|
enableNer: boolean;
|
|
329
336
|
enableConfidenceBoost: boolean;
|
|
330
337
|
enableCoreference: boolean;
|
|
@@ -387,7 +394,7 @@ declare const CURRENCY_PATTERN_META: Readonly<RegexMeta>;
|
|
|
387
394
|
declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, meta_: readonly RegexMeta[]) => Entity[];
|
|
388
395
|
//#endregion
|
|
389
396
|
//#region src/detectors/deny-list.d.ts
|
|
390
|
-
type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries">;
|
|
397
|
+
type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries" | "enableCountries">;
|
|
391
398
|
/**
|
|
392
399
|
* Source tag for each pattern in the automaton.
|
|
393
400
|
* "deny-list" = standard deny list entry
|
|
@@ -398,6 +405,8 @@ type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus"
|
|
|
398
405
|
* "title" = academic/professional title
|
|
399
406
|
*/
|
|
400
407
|
type PatternSource = "deny-list" | "city" | "custom-deny-list" | "first-name" | "surname" | "title";
|
|
408
|
+
type PatternLabels = string | string[];
|
|
409
|
+
type PatternSources = PatternSource | PatternSource[];
|
|
401
410
|
/**
|
|
402
411
|
* Pre-built deny list data. Constructed once by
|
|
403
412
|
* `buildDenyList`, reused across `processDenyListMatches`
|
|
@@ -411,10 +420,10 @@ type DenyListData = {
|
|
|
411
420
|
* appears in multiple dictionaries (e.g., "Denver"
|
|
412
421
|
* is both a person name and a city name).
|
|
413
422
|
*/
|
|
414
|
-
labels:
|
|
415
|
-
customLabels:
|
|
423
|
+
labels: PatternLabels[]; /** Maps pattern index → labels contributed by custom entries. */
|
|
424
|
+
customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
|
|
416
425
|
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
417
|
-
sources:
|
|
426
|
+
sources: PatternSources[];
|
|
418
427
|
};
|
|
419
428
|
/**
|
|
420
429
|
* Resolve which dictionaries to load based on country
|
|
@@ -452,6 +461,24 @@ declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Diction
|
|
|
452
461
|
*/
|
|
453
462
|
declare const processDenyListMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: DenyListData, ctx?: PipelineContext) => Entity[];
|
|
454
463
|
//#endregion
|
|
464
|
+
//#region src/detectors/countries.d.ts
|
|
465
|
+
/**
|
|
466
|
+
* Pre-built country patterns + parallel label/source
|
|
467
|
+
* metadata. Constructed once and reused across pipeline
|
|
468
|
+
* runs.
|
|
469
|
+
*/
|
|
470
|
+
type CountryData = {
|
|
471
|
+
/** Maps local pattern index to entity label. Always "country". */labels: string[];
|
|
472
|
+
/**
|
|
473
|
+
* Maps local pattern index to the alpha-2 ISO code the
|
|
474
|
+
* pattern resolves to. Used for downstream coreference /
|
|
475
|
+
* placeholder grouping.
|
|
476
|
+
*/
|
|
477
|
+
isoCodes: string[]; /** Maps local pattern index to pattern variant kind. */
|
|
478
|
+
variants: CountryVariant[];
|
|
479
|
+
};
|
|
480
|
+
type CountryVariant = "name" | "alias" | "alpha3" | "alpha2";
|
|
481
|
+
//#endregion
|
|
455
482
|
//#region src/build-unified-search.d.ts
|
|
456
483
|
type PatternSlice = {
|
|
457
484
|
start: number;
|
|
@@ -477,12 +504,14 @@ type UnifiedSearchInstance = {
|
|
|
477
504
|
denyList: PatternSlice;
|
|
478
505
|
streetTypes: PatternSlice;
|
|
479
506
|
gazetteer: PatternSlice;
|
|
507
|
+
countries: PatternSlice;
|
|
480
508
|
};
|
|
481
509
|
regexMeta: readonly RegexMeta[];
|
|
482
510
|
customRegexMeta: readonly RegexMeta[];
|
|
483
511
|
triggerRules: readonly TriggerRule[];
|
|
484
512
|
denyListData: DenyListData | null;
|
|
485
513
|
gazetteerData: GazetteerData | null;
|
|
514
|
+
countryData: CountryData | null;
|
|
486
515
|
};
|
|
487
516
|
declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: GazetteerEntry[], ctx?: PipelineContext) => Promise<UnifiedSearchInstance>;
|
|
488
517
|
//#endregion
|
|
@@ -702,18 +731,16 @@ declare const buildLegalFormPatterns: () => Promise<string[]>;
|
|
|
702
731
|
* otherwise the trim falls back to a no-op and sentence-
|
|
703
732
|
* fragment fixes do not apply.
|
|
704
733
|
*/
|
|
705
|
-
declare const processLegalFormMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText?: string
|
|
734
|
+
declare const processLegalFormMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText?: string, options?: {
|
|
735
|
+
suppressExtendBackward?: boolean;
|
|
736
|
+
}) => Entity[];
|
|
706
737
|
//#endregion
|
|
707
738
|
//#region src/detectors/triggers.d.ts
|
|
708
|
-
|
|
709
|
-
* Build trigger patterns and rules from data configs.
|
|
710
|
-
* Returns string[] for the unified TextSearch
|
|
711
|
-
* builder and the parallel rules array.
|
|
712
|
-
*/
|
|
713
|
-
declare const buildTriggerPatterns: () => Promise<{
|
|
739
|
+
type TriggerPatterns = {
|
|
714
740
|
patterns: string[];
|
|
715
741
|
rules: TriggerRule[];
|
|
716
|
-
}
|
|
742
|
+
};
|
|
743
|
+
declare const buildTriggerPatterns: () => Promise<TriggerPatterns>;
|
|
717
744
|
/**
|
|
718
745
|
* Process trigger matches from the unified search.
|
|
719
746
|
* Receives all matches; filters to the trigger slice
|
|
@@ -724,11 +751,6 @@ declare const buildTriggerPatterns: () => Promise<{
|
|
|
724
751
|
declare const processTriggerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, rules: readonly TriggerRule[]) => Entity[];
|
|
725
752
|
//#endregion
|
|
726
753
|
//#region src/detectors/address-seeds.d.ts
|
|
727
|
-
/**
|
|
728
|
-
* Build street type patterns for the unified search.
|
|
729
|
-
* Returns string[] for the unified TextSearch
|
|
730
|
-
* builder. Empty if data package is not installed.
|
|
731
|
-
*/
|
|
732
754
|
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
733
755
|
/**
|
|
734
756
|
* Process address seeds from the unified search.
|