@stll/anonymize-wasm 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-street-types.mjs +20 -1
- package/dist/allow-list.mjs +12 -0
- package/dist/amount-words.mjs +472 -36
- package/dist/constants.d.mts +2 -0
- package/dist/constants.mjs +85 -0
- package/dist/constants.mjs.map +1 -0
- package/dist/constants2.d.mts +65 -0
- package/dist/generic-roles.mjs +1 -0
- package/dist/rolldown-runtime.mjs +13 -0
- package/dist/structural-single-cap-prefixes.mjs +99 -0
- package/dist/structural-single-cap-prefixes.mjs.map +1 -0
- package/dist/triggers.en.mjs +48 -1
- package/dist/wasm.d.mts +12 -38
- package/dist/wasm.mjs +722 -156
- package/dist/wasm.mjs.map +1 -1
- package/package.json +6 -1
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
//#region src/constants.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Runtime-free constants for the anonymization pipeline.
|
|
4
|
+
*
|
|
5
|
+
* This module is the SSR-safe / browser-safe entrypoint:
|
|
6
|
+
* importing it must not pull in `@stll/text-search`,
|
|
7
|
+
* `@stll/anonymize-wasm`, or any other runtime-bearing
|
|
8
|
+
* module. Consumers that only need the static label list,
|
|
9
|
+
* detection-source identifiers, or operator names import
|
|
10
|
+
* from `@stll/anonymize/constants` (or
|
|
11
|
+
* `@stll/anonymize-wasm/constants`) without paying the
|
|
12
|
+
* wasm / regex-set startup cost.
|
|
13
|
+
*
|
|
14
|
+
* `types.ts` re-exports these for back-compat, so existing
|
|
15
|
+
* `import { DEFAULT_ENTITY_LABELS } from "@stll/anonymize"`
|
|
16
|
+
* call sites keep working.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Source of a detected entity span.
|
|
20
|
+
* Ordered by detection layer in the pipeline.
|
|
21
|
+
*/
|
|
22
|
+
declare const DETECTION_SOURCES: {
|
|
23
|
+
readonly TRIGGER: "trigger";
|
|
24
|
+
readonly REGEX: "regex";
|
|
25
|
+
readonly DENY_LIST: "deny-list";
|
|
26
|
+
readonly LEGAL_FORM: "legal-form";
|
|
27
|
+
readonly GAZETTEER: "gazetteer";
|
|
28
|
+
readonly NER: "ner";
|
|
29
|
+
readonly COREFERENCE: "coreference";
|
|
30
|
+
};
|
|
31
|
+
type DetectionSource = (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];
|
|
32
|
+
/**
|
|
33
|
+
* Priority levels for detection sources.
|
|
34
|
+
* Higher = more structurally reliable. Used during
|
|
35
|
+
* overlap resolution so deterministic detectors beat
|
|
36
|
+
* probabilistic ones regardless of raw score.
|
|
37
|
+
*/
|
|
38
|
+
declare const DETECTOR_PRIORITY: {
|
|
39
|
+
readonly gazetteer: 5;
|
|
40
|
+
readonly trigger: 4;
|
|
41
|
+
readonly "legal-form": 3;
|
|
42
|
+
readonly regex: 3;
|
|
43
|
+
readonly "deny-list": 2;
|
|
44
|
+
readonly coreference: 2;
|
|
45
|
+
readonly ner: 1;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Anonymization operator types. Each operator defines
|
|
49
|
+
* how a confirmed entity is replaced in the output.
|
|
50
|
+
*/
|
|
51
|
+
declare const OPERATOR_TYPES: readonly ["replace", "redact"];
|
|
52
|
+
type OperatorType = (typeof OPERATOR_TYPES)[number];
|
|
53
|
+
/**
|
|
54
|
+
* Canonical entity labels used across the pipeline.
|
|
55
|
+
* NER models may use different native labels; the bench
|
|
56
|
+
* NER wrapper maps model output to these canonical names.
|
|
57
|
+
*
|
|
58
|
+
* These labels are ephemeral: entities are regenerated on
|
|
59
|
+
* every pipeline run and never persisted to the database.
|
|
60
|
+
* Renaming a label here requires no migration.
|
|
61
|
+
*/
|
|
62
|
+
declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "birth number", "national identification number", "social security number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel", "misc"];
|
|
63
|
+
//#endregion
|
|
64
|
+
export { OPERATOR_TYPES as a, DetectionSource as i, DETECTION_SOURCES as n, OperatorType as o, DETECTOR_PRIORITY as r, DEFAULT_ENTITY_LABELS as t };
|
|
65
|
+
//# sourceMappingURL=constants2.d.mts.map
|
package/dist/generic-roles.mjs
CHANGED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
//#region \0rolldown/runtime.js
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __exportAll = (all, no_symbols) => {
|
|
4
|
+
let target = {};
|
|
5
|
+
for (var name in all) __defProp(target, name, {
|
|
6
|
+
get: all[name],
|
|
7
|
+
enumerable: true
|
|
8
|
+
});
|
|
9
|
+
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
10
|
+
return target;
|
|
11
|
+
};
|
|
12
|
+
//#endregion
|
|
13
|
+
export { __exportAll as t };
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
//#region src/data/structural-single-cap-prefixes.json
|
|
2
|
+
var _comment = "Structural document-unit words that can precede a single-letter or letter-number marker before a legal form word, e.g. 'Schedule A LLC Members' or 'Exhibit A-1 LLC Agreement'. Lowercased; matched case-insensitively. Organised per-language; runtime unions all languages because match spans do not carry a language tag.";
|
|
3
|
+
var en = [
|
|
4
|
+
"annex",
|
|
5
|
+
"appendix",
|
|
6
|
+
"article",
|
|
7
|
+
"attachment",
|
|
8
|
+
"clause",
|
|
9
|
+
"exhibit",
|
|
10
|
+
"paragraph",
|
|
11
|
+
"part",
|
|
12
|
+
"schedule",
|
|
13
|
+
"section"
|
|
14
|
+
];
|
|
15
|
+
var cs = [
|
|
16
|
+
"bod",
|
|
17
|
+
"paragraf",
|
|
18
|
+
"podbod",
|
|
19
|
+
"ustanovení",
|
|
20
|
+
"článek"
|
|
21
|
+
];
|
|
22
|
+
var sk = [
|
|
23
|
+
"bod",
|
|
24
|
+
"hlava",
|
|
25
|
+
"oddiel",
|
|
26
|
+
"odsek",
|
|
27
|
+
"pododdiel",
|
|
28
|
+
"sekcia",
|
|
29
|
+
"časť",
|
|
30
|
+
"článok"
|
|
31
|
+
];
|
|
32
|
+
var de = [
|
|
33
|
+
"absatz",
|
|
34
|
+
"abschnitt",
|
|
35
|
+
"abteilung",
|
|
36
|
+
"anlage",
|
|
37
|
+
"anhang",
|
|
38
|
+
"artikel",
|
|
39
|
+
"kapitel",
|
|
40
|
+
"klausel",
|
|
41
|
+
"punkt",
|
|
42
|
+
"sektion",
|
|
43
|
+
"unterabschnitt",
|
|
44
|
+
"unterpunkt"
|
|
45
|
+
];
|
|
46
|
+
var es = [
|
|
47
|
+
"anexo",
|
|
48
|
+
"apartado",
|
|
49
|
+
"artículo",
|
|
50
|
+
"cláusula",
|
|
51
|
+
"sección"
|
|
52
|
+
];
|
|
53
|
+
var fr = [
|
|
54
|
+
"annexe",
|
|
55
|
+
"article",
|
|
56
|
+
"clause",
|
|
57
|
+
"paragraphe",
|
|
58
|
+
"partie",
|
|
59
|
+
"section"
|
|
60
|
+
];
|
|
61
|
+
var it = [
|
|
62
|
+
"allegato",
|
|
63
|
+
"appendice",
|
|
64
|
+
"articolo",
|
|
65
|
+
"clausola",
|
|
66
|
+
"paragrafo",
|
|
67
|
+
"sezione"
|
|
68
|
+
];
|
|
69
|
+
var pl = [
|
|
70
|
+
"aneksem",
|
|
71
|
+
"artykuł",
|
|
72
|
+
"paragraf",
|
|
73
|
+
"punkt",
|
|
74
|
+
"sekcja",
|
|
75
|
+
"załącznik"
|
|
76
|
+
];
|
|
77
|
+
var pt_br = [
|
|
78
|
+
"anexo",
|
|
79
|
+
"artigo",
|
|
80
|
+
"cláusula",
|
|
81
|
+
"parágrafo",
|
|
82
|
+
"seção"
|
|
83
|
+
];
|
|
84
|
+
var structural_single_cap_prefixes_default = {
|
|
85
|
+
_comment,
|
|
86
|
+
en,
|
|
87
|
+
cs,
|
|
88
|
+
sk,
|
|
89
|
+
de,
|
|
90
|
+
es,
|
|
91
|
+
fr,
|
|
92
|
+
it,
|
|
93
|
+
pl,
|
|
94
|
+
"pt-br": pt_br
|
|
95
|
+
};
|
|
96
|
+
//#endregion
|
|
97
|
+
export { _comment, cs, de, structural_single_cap_prefixes_default as default, en, es, fr, it, pl, pt_br as "pt-br", sk };
|
|
98
|
+
|
|
99
|
+
//# sourceMappingURL=structural-single-cap-prefixes.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"structural-single-cap-prefixes.mjs","names":[],"sources":["../../src/data/structural-single-cap-prefixes.json"],"sourcesContent":[""],"mappings":""}
|
package/dist/triggers.en.mjs
CHANGED
|
@@ -33,7 +33,22 @@ var triggers_en_default = [
|
|
|
33
33
|
{
|
|
34
34
|
"id": "en-jurisdiction",
|
|
35
35
|
"label": "address",
|
|
36
|
-
"strategy": {
|
|
36
|
+
"strategy": {
|
|
37
|
+
"type": "to-next-comma",
|
|
38
|
+
"maxLength": 30,
|
|
39
|
+
"stopWords": [
|
|
40
|
+
"or",
|
|
41
|
+
"and",
|
|
42
|
+
"in",
|
|
43
|
+
"sitting",
|
|
44
|
+
"shall",
|
|
45
|
+
"that",
|
|
46
|
+
"which",
|
|
47
|
+
"without",
|
|
48
|
+
"for",
|
|
49
|
+
"to"
|
|
50
|
+
]
|
|
51
|
+
},
|
|
37
52
|
"triggers": [
|
|
38
53
|
"State of",
|
|
39
54
|
"Commonwealth of",
|
|
@@ -64,6 +79,38 @@ var triggers_en_default = [
|
|
|
64
79
|
"tax identification number",
|
|
65
80
|
"tax id"
|
|
66
81
|
]
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"id": "en-uk-companies-house",
|
|
85
|
+
"label": "registration number",
|
|
86
|
+
"strategy": { "type": "company-id-value" },
|
|
87
|
+
"triggers": [
|
|
88
|
+
"Companies House",
|
|
89
|
+
"Company No.",
|
|
90
|
+
"Company No",
|
|
91
|
+
"CRN"
|
|
92
|
+
],
|
|
93
|
+
"extensions": ["add-colon", "add-colon-space"],
|
|
94
|
+
"validations": [{
|
|
95
|
+
"type": "matches-pattern",
|
|
96
|
+
"pattern": "^(?:SC|NI|OC|SO|R|FC|GE|GS|IC|IP|LP|NC|NF|NL|NO|NP|NR|NZ|RC|SA|SF|SI|SL|SP|SR|SZ)?\\d{6,8}$"
|
|
97
|
+
}]
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "en-uk-utr",
|
|
101
|
+
"label": "tax identification number",
|
|
102
|
+
"strategy": { "type": "company-id-value" },
|
|
103
|
+
"triggers": [
|
|
104
|
+
"UTR",
|
|
105
|
+
"Unique Tax Reference",
|
|
106
|
+
"Unique Taxpayer Reference",
|
|
107
|
+
"Self Assessment UTR"
|
|
108
|
+
],
|
|
109
|
+
"extensions": ["add-colon", "add-colon-space"],
|
|
110
|
+
"validations": [{
|
|
111
|
+
"type": "matches-pattern",
|
|
112
|
+
"pattern": "^\\d{10}$"
|
|
113
|
+
}]
|
|
67
114
|
}
|
|
68
115
|
];
|
|
69
116
|
//#endregion
|
package/dist/wasm.d.mts
CHANGED
|
@@ -1,29 +1,9 @@
|
|
|
1
|
+
import { a as OPERATOR_TYPES, i as DetectionSource, n as DETECTION_SOURCES, o as OperatorType, r as DETECTOR_PRIORITY, t as DEFAULT_ENTITY_LABELS } from "./constants2.mjs";
|
|
1
2
|
import { Validator } from "@stll/stdnum";
|
|
2
3
|
import { Match, PatternEntry, TextSearch } from "@stll/text-search";
|
|
3
4
|
import { Tokenizer } from "@huggingface/tokenizers";
|
|
4
5
|
|
|
5
6
|
//#region src/types.d.ts
|
|
6
|
-
/**
|
|
7
|
-
* Source of a detected entity span.
|
|
8
|
-
* Ordered by detection layer in the pipeline.
|
|
9
|
-
*/
|
|
10
|
-
declare const DETECTION_SOURCES: {
|
|
11
|
-
readonly TRIGGER: "trigger";
|
|
12
|
-
readonly REGEX: "regex";
|
|
13
|
-
readonly DENY_LIST: "deny-list";
|
|
14
|
-
readonly LEGAL_FORM: "legal-form";
|
|
15
|
-
readonly GAZETTEER: "gazetteer";
|
|
16
|
-
readonly NER: "ner";
|
|
17
|
-
readonly COREFERENCE: "coreference";
|
|
18
|
-
};
|
|
19
|
-
type DetectionSource = (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];
|
|
20
|
-
/**
|
|
21
|
-
* Priority levels for detection sources.
|
|
22
|
-
* Higher = more structurally reliable. Used during
|
|
23
|
-
* overlap resolution so deterministic detectors beat
|
|
24
|
-
* probabilistic ones regardless of raw score.
|
|
25
|
-
*/
|
|
26
|
-
declare const DETECTOR_PRIORITY: Record<DetectionSource, number>;
|
|
27
7
|
/**
|
|
28
8
|
* A detected PII entity span in the source text.
|
|
29
9
|
* Every detection layer produces these.
|
|
@@ -35,7 +15,7 @@ type Entity = {
|
|
|
35
15
|
text: string;
|
|
36
16
|
score: number;
|
|
37
17
|
source: DetectionSource;
|
|
38
|
-
sourceDetail?: "custom-deny-list" | "custom-regex";
|
|
18
|
+
sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
|
|
39
19
|
};
|
|
40
20
|
/**
|
|
41
21
|
* Entity after human review. Extends the base Entity
|
|
@@ -72,6 +52,16 @@ type TriggerStrategy = {
|
|
|
72
52
|
* word-boundary, case-insensitive.
|
|
73
53
|
*/
|
|
74
54
|
stopWords?: string[];
|
|
55
|
+
/**
|
|
56
|
+
* Hard cap on the captured span length, in characters,
|
|
57
|
+
* regardless of where the next comma / stop char sits.
|
|
58
|
+
* Use for triggers that label short formulaic phrases
|
|
59
|
+
* ("State of Delaware") and must not absorb the rest
|
|
60
|
+
* of a long forum-selection clause when the comma is
|
|
61
|
+
* sentences away. Falls back to the default 100-char
|
|
62
|
+
* fallback when omitted.
|
|
63
|
+
*/
|
|
64
|
+
maxLength?: number;
|
|
75
65
|
} | {
|
|
76
66
|
type: "to-end-of-line";
|
|
77
67
|
} | {
|
|
@@ -176,12 +166,6 @@ type TriggerRule = {
|
|
|
176
166
|
validations: CompiledValidation[];
|
|
177
167
|
includeTrigger: boolean;
|
|
178
168
|
};
|
|
179
|
-
/**
|
|
180
|
-
* Anonymisation operator types. Each operator defines
|
|
181
|
-
* how a confirmed entity is replaced in the output.
|
|
182
|
-
*/
|
|
183
|
-
declare const OPERATOR_TYPES: readonly ["replace", "redact"];
|
|
184
|
-
type OperatorType = (typeof OPERATOR_TYPES)[number];
|
|
185
169
|
/** Per-label operator selection. Key is the entity label. */
|
|
186
170
|
type OperatorConfig = {
|
|
187
171
|
/** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
|
|
@@ -341,16 +325,6 @@ type PipelineConfig = {
|
|
|
341
325
|
*/
|
|
342
326
|
dictionaries?: Dictionaries;
|
|
343
327
|
};
|
|
344
|
-
/**
|
|
345
|
-
* Canonical entity labels used across the pipeline.
|
|
346
|
-
* NER models may use different native labels; the bench
|
|
347
|
-
* NER wrapper maps model output to these canonical names.
|
|
348
|
-
*
|
|
349
|
-
* These labels are ephemeral: entities are regenerated on
|
|
350
|
-
* every pipeline run and never persisted to the database.
|
|
351
|
-
* Renaming a label here requires no migration.
|
|
352
|
-
*/
|
|
353
|
-
declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "birth number", "national identification number", "social security number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel", "misc"];
|
|
354
328
|
//#endregion
|
|
355
329
|
//#region src/detectors/regex.d.ts
|
|
356
330
|
type RegexMeta = {
|