@stll/anonymize-wasm 2.0.0-alpha.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -4
- package/dist/native/index.wasi-browser.js +12002 -0
- package/dist/native/index.wasi.cjs +122 -0
- package/dist/native/index.wasm32-wasi.wasm +0 -0
- package/dist/native/native-pipeline.cs.stlanonpkg +0 -0
- package/dist/native/native-pipeline.de.stlanonpkg +0 -0
- package/dist/native/native-pipeline.en.stlanonpkg +0 -0
- package/dist/native/native-pipeline.stlanonpkg +0 -0
- package/dist/native/wasi-worker-browser.mjs +10766 -0
- package/dist/native/wasi-worker.mjs +63 -0
- package/dist/vite.d.mts +18 -2
- package/dist/vite.mjs +89 -12
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +181 -1356
- package/dist/wasm.mjs +185 -18644
- package/dist/wasm.mjs.map +1 -1
- package/package.json +15 -9
- package/dist/address-boundaries.mjs +0 -197
- package/dist/address-boundaries.mjs.map +0 -1
- package/dist/address-jurisdiction-prefixes.mjs +0 -16
- package/dist/address-jurisdiction-prefixes.mjs.map +0 -1
- package/dist/address-stop-keywords.mjs +0 -148
- package/dist/address-stop-keywords.mjs.map +0 -1
- package/dist/address-stopwords.mjs +0 -84
- package/dist/address-stopwords.mjs.map +0 -1
- package/dist/address-unit-abbreviations.mjs +0 -15
- package/dist/address-unit-abbreviations.mjs.map +0 -1
- package/dist/allow-list.mjs +0 -196
- package/dist/allow-list.mjs.map +0 -1
- package/dist/clause-noun-heads.mjs +0 -79
- package/dist/clause-noun-heads.mjs.map +0 -1
- package/dist/common-words-en.mjs +0 -9887
- package/dist/common-words-en.mjs.map +0 -1
- package/dist/coreference-org-determiners.mjs +0 -19
- package/dist/coreference-org-determiners.mjs.map +0 -1
- package/dist/coreference.cs.mjs +0 -14
- package/dist/coreference.cs.mjs.map +0 -1
- package/dist/coreference.de.mjs +0 -14
- package/dist/coreference.de.mjs.map +0 -1
- package/dist/coreference.en.mjs +0 -14
- package/dist/coreference.en.mjs.map +0 -1
- package/dist/coreference.es.mjs +0 -22
- package/dist/coreference.es.mjs.map +0 -1
- package/dist/coreference.fr.mjs +0 -32
- package/dist/coreference.fr.mjs.map +0 -1
- package/dist/coreference.it.mjs +0 -27
- package/dist/coreference.it.mjs.map +0 -1
- package/dist/coreference.pl.mjs +0 -27
- package/dist/coreference.pl.mjs.map +0 -1
- package/dist/coreference.pt-br.mjs +0 -14
- package/dist/coreference.pt-br.mjs.map +0 -1
- package/dist/coreference.sk.mjs +0 -27
- package/dist/coreference.sk.mjs.map +0 -1
- package/dist/currencies.mjs +0 -231
- package/dist/currencies.mjs.map +0 -1
- package/dist/date-months.mjs +0 -618
- package/dist/date-months.mjs.map +0 -1
- package/dist/defined-term-heads.mjs +0 -15
- package/dist/defined-term-heads.mjs.map +0 -1
- package/dist/document-structure-headings.mjs +0 -90
- package/dist/document-structure-headings.mjs.map +0 -1
- package/dist/false-positive-shapes.mjs +0 -36
- package/dist/false-positive-shapes.mjs.map +0 -1
- package/dist/generic-roles.mjs +0 -244
- package/dist/generic-roles.mjs.map +0 -1
- package/dist/hotword-rules.mjs +0 -149
- package/dist/hotword-rules.mjs.map +0 -1
- package/dist/legal-form-leading-clauses.mjs +0 -23
- package/dist/legal-form-leading-clauses.mjs.map +0 -1
- package/dist/legal-forms.mjs +0 -2115
- package/dist/legal-forms.mjs.map +0 -1
- package/dist/legal-role-heads.cs.mjs +0 -48
- package/dist/legal-role-heads.cs.mjs.map +0 -1
- package/dist/legal-role-heads.de.mjs +0 -33
- package/dist/legal-role-heads.de.mjs.map +0 -1
- package/dist/legal-role-heads.en.mjs +0 -37
- package/dist/legal-role-heads.en.mjs.map +0 -1
- package/dist/legal-role-heads.es.mjs +0 -54
- package/dist/legal-role-heads.es.mjs.map +0 -1
- package/dist/legal-role-heads.fr.mjs +0 -72
- package/dist/legal-role-heads.fr.mjs.map +0 -1
- package/dist/legal-role-heads.it.mjs +0 -68
- package/dist/legal-role-heads.it.mjs.map +0 -1
- package/dist/legal-role-heads.pl.mjs +0 -84
- package/dist/legal-role-heads.pl.mjs.map +0 -1
- package/dist/legal-role-heads.pt-br.mjs +0 -63
- package/dist/legal-role-heads.pt-br.mjs.map +0 -1
- package/dist/legal-role-heads.sk.mjs +0 -80
- package/dist/legal-role-heads.sk.mjs.map +0 -1
- package/dist/manifest.mjs +0 -69
- package/dist/manifest.mjs.map +0 -1
- package/dist/names-exclusions.mjs +0 -223
- package/dist/names-exclusions.mjs.map +0 -1
- package/dist/names-first.mjs +0 -418
- package/dist/names-first.mjs.map +0 -1
- package/dist/names-nw-ar.mjs +0 -202
- package/dist/names-nw-ar.mjs.map +0 -1
- package/dist/names-nw-excluded-allcaps.mjs +0 -112
- package/dist/names-nw-excluded-allcaps.mjs.map +0 -1
- package/dist/names-nw-fil.mjs +0 -202
- package/dist/names-nw-fil.mjs.map +0 -1
- package/dist/names-nw-id.mjs +0 -210
- package/dist/names-nw-id.mjs.map +0 -1
- package/dist/names-nw-in.mjs +0 -526
- package/dist/names-nw-in.mjs.map +0 -1
- package/dist/names-nw-ja-latn.mjs +0 -260
- package/dist/names-nw-ja-latn.mjs.map +0 -1
- package/dist/names-nw-ko.mjs +0 -162
- package/dist/names-nw-ko.mjs.map +0 -1
- package/dist/names-nw-th.mjs +0 -188
- package/dist/names-nw-th.mjs.map +0 -1
- package/dist/names-nw-vi.mjs +0 -151
- package/dist/names-nw-vi.mjs.map +0 -1
- package/dist/names-nw-zh-latn.mjs +0 -197
- package/dist/names-nw-zh-latn.mjs.map +0 -1
- package/dist/names-surnames.mjs +0 -113
- package/dist/names-surnames.mjs.map +0 -1
- package/dist/names-title-tokens.mjs +0 -40
- package/dist/names-title-tokens.mjs.map +0 -1
- package/dist/organization-unit-heads.mjs +0 -20
- package/dist/organization-unit-heads.mjs.map +0 -1
- package/dist/person-stopwords.mjs +0 -211
- package/dist/person-stopwords.mjs.map +0 -1
- package/dist/section-headings.mjs +0 -64
- package/dist/section-headings.mjs.map +0 -1
- package/dist/sentence-verb-indicators.mjs +0 -232
- package/dist/sentence-verb-indicators.mjs.map +0 -1
- package/dist/signing-clauses.mjs +0 -102
- package/dist/signing-clauses.mjs.map +0 -1
- package/dist/stopwords.mjs +0 -9915
- package/dist/stopwords.mjs.map +0 -1
- package/dist/structural-single-cap-prefixes.mjs +0 -99
- package/dist/structural-single-cap-prefixes.mjs.map +0 -1
- package/dist/triggers.cs.mjs +0 -569
- package/dist/triggers.cs.mjs.map +0 -1
- package/dist/triggers.de.mjs +0 -139
- package/dist/triggers.de.mjs.map +0 -1
- package/dist/triggers.en.mjs +0 -119
- package/dist/triggers.en.mjs.map +0 -1
- package/dist/triggers.es.mjs +0 -96
- package/dist/triggers.es.mjs.map +0 -1
- package/dist/triggers.fr.mjs +0 -275
- package/dist/triggers.fr.mjs.map +0 -1
- package/dist/triggers.global.mjs +0 -79
- package/dist/triggers.global.mjs.map +0 -1
- package/dist/triggers.hu.mjs +0 -41
- package/dist/triggers.hu.mjs.map +0 -1
- package/dist/triggers.it.mjs +0 -74
- package/dist/triggers.it.mjs.map +0 -1
- package/dist/triggers.pl.mjs +0 -271
- package/dist/triggers.pl.mjs.map +0 -1
- package/dist/triggers.pt-br.mjs +0 -193
- package/dist/triggers.pt-br.mjs.map +0 -1
- package/dist/triggers.ro.mjs +0 -59
- package/dist/triggers.ro.mjs.map +0 -1
- package/dist/triggers.sk.mjs +0 -555
- package/dist/triggers.sk.mjs.map +0 -1
- package/dist/triggers.sv.mjs +0 -58
- package/dist/triggers.sv.mjs.map +0 -1
- package/dist/year-words.mjs +0 -62
- package/dist/year-words.mjs.map +0 -1
package/dist/wasm.d.mts
CHANGED
|
@@ -1,625 +1,18 @@
|
|
|
1
1
|
import { a as OPERATOR_TYPES, i as DetectionSource, n as DETECTION_SOURCES, o as OperatorType, r as DETECTOR_PRIORITY, t as DEFAULT_ENTITY_LABELS } from "./constants2.mjs";
|
|
2
|
-
import { Validator } from "@stll/stdnum";
|
|
3
|
-
import { Match, PatternEntry, TextSearch } from "@stll/text-search";
|
|
4
|
-
import { Tokenizer } from "@huggingface/tokenizers";
|
|
5
2
|
|
|
6
|
-
//#region src/
|
|
7
|
-
/**
|
|
8
|
-
* Fields shared by every entity span in the source text.
|
|
9
|
-
*/
|
|
10
|
-
type EntityBase = {
|
|
11
|
-
start: number;
|
|
12
|
-
end: number;
|
|
13
|
-
label: string;
|
|
14
|
-
text: string;
|
|
15
|
-
score: number;
|
|
16
|
-
sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
|
|
17
|
-
};
|
|
18
|
-
/**
|
|
19
|
-
* A PII entity span found by a primary detection layer
|
|
20
|
-
* (regex, NER, legal forms, deny list, ...).
|
|
21
|
-
*/
|
|
22
|
-
type DetectedEntity = EntityBase & {
|
|
23
|
-
source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
|
|
24
|
-
};
|
|
3
|
+
//#region src/native-search-config.d.ts
|
|
25
4
|
/**
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* mention ("Acme" after "Acme Corp.").
|
|
5
|
+
* Structural type for the prepared static-search config the native binding
|
|
6
|
+
* consumes and the Rust assembler emits (`assembleStaticSearchConfigJson`).
|
|
29
7
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*/
|
|
38
|
-
type CorefAliasEntity = EntityBase & {
|
|
39
|
-
source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
|
|
40
|
-
corefSourceText: string;
|
|
41
|
-
};
|
|
42
|
-
/**
|
|
43
|
-
* A detected PII entity span in the source text.
|
|
44
|
-
* Every detection layer produces these.
|
|
45
|
-
*/
|
|
46
|
-
type Entity = DetectedEntity | CorefAliasEntity;
|
|
47
|
-
/**
|
|
48
|
-
* Entity after human review. Extends the base Entity
|
|
49
|
-
* with a review decision.
|
|
50
|
-
*/
|
|
51
|
-
type ReviewDecision = "confirmed" | "rejected" | "relabeled";
|
|
52
|
-
type ReviewedEntity = Entity & {
|
|
53
|
-
decision?: ReviewDecision;
|
|
54
|
-
originalLabel?: string;
|
|
55
|
-
};
|
|
56
|
-
/**
|
|
57
|
-
* A single entry in the workspace-scoped gazetteer
|
|
58
|
-
* (deny list). Persisted in IndexedDB.
|
|
8
|
+
* This config used to be built in TypeScript by `build-unified-search.ts`; that
|
|
9
|
+
* layer was retired in favor of the Rust assembler
|
|
10
|
+
* (`crates/anonymize-adapter-contract` `assemble_static_search_config`). The
|
|
11
|
+
* type now lives here as a pure, dependency-free description of the JSON the
|
|
12
|
+
* binding accepts on its `fromConfigJsonBytes` / prepare paths, so callers that
|
|
13
|
+
* hold a pre-assembled config keep a precise type without pulling in the
|
|
14
|
+
* deleted detector modules.
|
|
59
15
|
*/
|
|
60
|
-
type GazetteerEntry = {
|
|
61
|
-
id: string;
|
|
62
|
-
canonical: string;
|
|
63
|
-
label: string;
|
|
64
|
-
variants: string[];
|
|
65
|
-
workspaceId: string;
|
|
66
|
-
createdAt: number;
|
|
67
|
-
source: "manual" | "confirmed-from-model";
|
|
68
|
-
};
|
|
69
|
-
/** Extraction strategy — closed discriminated union. */
|
|
70
|
-
type TriggerStrategy = {
|
|
71
|
-
type: "to-next-comma";
|
|
72
|
-
/**
|
|
73
|
-
* Optional list of lowercase keywords that terminate
|
|
74
|
-
* the value scan, in addition to commas/newlines. Useful
|
|
75
|
-
* for triggers like court names that may continue past
|
|
76
|
-
* a missing comma into adjacent clause text ("Městským
|
|
77
|
-
* soudem v Praze dne 1. 1. 2020"); listing `"dne"` here
|
|
78
|
-
* stops the scan at the date boundary. Matched on a
|
|
79
|
-
* word-boundary, case-insensitive.
|
|
80
|
-
*/
|
|
81
|
-
stopWords?: string[];
|
|
82
|
-
/**
|
|
83
|
-
* Hard cap on the captured span length, in characters,
|
|
84
|
-
* regardless of where the next comma / stop char sits.
|
|
85
|
-
* Use for triggers that label short formulaic phrases
|
|
86
|
-
* ("State of Delaware") and must not absorb the rest
|
|
87
|
-
* of a long forum-selection clause when the comma is
|
|
88
|
-
* sentences away. Falls back to the default 100-char
|
|
89
|
-
* fallback when omitted.
|
|
90
|
-
*/
|
|
91
|
-
maxLength?: number;
|
|
92
|
-
} | {
|
|
93
|
-
type: "to-end-of-line";
|
|
94
|
-
} | {
|
|
95
|
-
type: "n-words";
|
|
96
|
-
count: number;
|
|
97
|
-
} | {
|
|
98
|
-
type: "company-id-value";
|
|
99
|
-
} | {
|
|
100
|
-
type: "address";
|
|
101
|
-
maxChars?: number;
|
|
102
|
-
} | {
|
|
103
|
-
/**
|
|
104
|
-
* Extract the first regex match in the value text.
|
|
105
|
-
* Useful for shape-bounded values that follow a
|
|
106
|
-
* label on the same line as other fields, where
|
|
107
|
-
* `to-end-of-line` would over-capture. The pattern
|
|
108
|
-
* is anchored to the start of the (already
|
|
109
|
-
* leading-whitespace-stripped) value, so use
|
|
110
|
-
* `(?:.*?)` prefix only when intentional.
|
|
111
|
-
*/
|
|
112
|
-
type: "match-pattern";
|
|
113
|
-
pattern: string;
|
|
114
|
-
flags?: string;
|
|
115
|
-
};
|
|
116
|
-
/** Validation rules — closed discriminated union. */
|
|
117
|
-
type TriggerValidation = {
|
|
118
|
-
type: "starts-uppercase";
|
|
119
|
-
} | {
|
|
120
|
-
type: "min-length";
|
|
121
|
-
min: number;
|
|
122
|
-
} | {
|
|
123
|
-
type: "max-length";
|
|
124
|
-
max: number;
|
|
125
|
-
} | {
|
|
126
|
-
type: "no-digits";
|
|
127
|
-
} | {
|
|
128
|
-
type: "has-digits";
|
|
129
|
-
} | {
|
|
130
|
-
type: "matches-pattern";
|
|
131
|
-
pattern: string;
|
|
132
|
-
flags?: string;
|
|
133
|
-
}
|
|
134
|
-
/**
|
|
135
|
-
* Run a named stdnum validator (checksum + length)
|
|
136
|
-
* against the captured value. Keeps the trigger
|
|
137
|
-
* path symmetrical with the formatted-regex
|
|
138
|
-
* detectors so e.g. `CPF nº 00000000000` does not
|
|
139
|
-
* survive as a tax-ID entity.
|
|
140
|
-
*/
|
|
141
|
-
| {
|
|
142
|
-
type: "valid-id";
|
|
143
|
-
validator: ValidIdValidator;
|
|
144
|
-
};
|
|
145
|
-
/** Built-in stdnum validators that can be referenced
|
|
146
|
-
* by `valid-id` validations. */
|
|
147
|
-
type ValidIdValidator = "br.cpf" | "br.cnpj" | "us.rtn";
|
|
148
|
-
/** Auto-generated trigger variants — closed set. */
|
|
149
|
-
type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
|
|
150
|
-
/** V2 trigger config entry (JSON shape). */
|
|
151
|
-
type TriggerGroupConfig = {
|
|
152
|
-
id?: string;
|
|
153
|
-
triggers: string[];
|
|
154
|
-
label: string;
|
|
155
|
-
strategy: TriggerStrategy;
|
|
156
|
-
extensions?: TriggerExtension[];
|
|
157
|
-
validations?: TriggerValidation[];
|
|
158
|
-
/** When true, include the trigger text in the
|
|
159
|
-
* entity span (e.g., court names). */
|
|
160
|
-
includeTrigger?: boolean;
|
|
161
|
-
};
|
|
162
|
-
/** Compiled validation with pre-built regex. */
|
|
163
|
-
type CompiledValidation = {
|
|
164
|
-
type: "starts-uppercase";
|
|
165
|
-
re: RegExp;
|
|
166
|
-
} | {
|
|
167
|
-
type: "min-length";
|
|
168
|
-
min: number;
|
|
169
|
-
} | {
|
|
170
|
-
type: "max-length";
|
|
171
|
-
max: number;
|
|
172
|
-
} | {
|
|
173
|
-
type: "no-digits";
|
|
174
|
-
re: RegExp;
|
|
175
|
-
} | {
|
|
176
|
-
type: "has-digits";
|
|
177
|
-
re: RegExp;
|
|
178
|
-
} | {
|
|
179
|
-
type: "matches-pattern";
|
|
180
|
-
re: RegExp;
|
|
181
|
-
} | {
|
|
182
|
-
type: "valid-id";
|
|
183
|
-
validator: ValidIdValidator;
|
|
184
|
-
check: (value: string) => boolean;
|
|
185
|
-
};
|
|
186
|
-
/**
|
|
187
|
-
* Runtime rule — one per trigger string after
|
|
188
|
-
* expansion. Fed to the Aho-Corasick automaton.
|
|
189
|
-
*/
|
|
190
|
-
type TriggerRule = {
|
|
191
|
-
trigger: string;
|
|
192
|
-
label: string;
|
|
193
|
-
strategy: TriggerStrategy;
|
|
194
|
-
validations: CompiledValidation[];
|
|
195
|
-
includeTrigger: boolean;
|
|
196
|
-
};
|
|
197
|
-
/** Per-label operator selection. Key is the entity label. */
|
|
198
|
-
type OperatorConfig = {
|
|
199
|
-
/** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
|
|
200
|
-
redactString: string;
|
|
201
|
-
};
|
|
202
|
-
/** Whether an operator produces a reversible redaction entry. */
|
|
203
|
-
type OperatorReversibility = "reversible" | "irreversible";
|
|
204
|
-
type AnonymisationOperator = {
|
|
205
|
-
type: OperatorType;
|
|
206
|
-
reversibility: OperatorReversibility;
|
|
207
|
-
/**
|
|
208
|
-
* Apply the operator to a single entity occurrence.
|
|
209
|
-
* Returns the replacement string to embed in the document.
|
|
210
|
-
*/
|
|
211
|
-
apply: (text: string, label: string, placeholder: string, redactString: string) => string;
|
|
212
|
-
};
|
|
213
|
-
/**
|
|
214
|
-
* Redacted document output with stable entity mapping.
|
|
215
|
-
*/
|
|
216
|
-
type RedactionResult = {
|
|
217
|
-
redactedText: string;
|
|
218
|
-
/**
|
|
219
|
-
* Maps placeholder to original text. Only populated for
|
|
220
|
-
* reversible operators (replace). Empty for redact.
|
|
221
|
-
*/
|
|
222
|
-
redactionMap: Map<string, string>; /** Maps placeholder to the operator that produced it. */
|
|
223
|
-
operatorMap: Map<string, OperatorType>;
|
|
224
|
-
entityCount: number;
|
|
225
|
-
};
|
|
226
|
-
/**
|
|
227
|
-
* Configuration for the detection pipeline.
|
|
228
|
-
*/
|
|
229
|
-
type DenyListCategory = "Names" | "Places" | "Addresses" | "Courts" | "Financial" | "Government" | "Healthcare" | "Education" | "Political" | "Organizations" | "International";
|
|
230
|
-
/**
|
|
231
|
-
* Metadata for a single dictionary entry in the
|
|
232
|
-
* deny-list system. Mirrors the shape from
|
|
233
|
-
* the anonymize-data package so consumers can pass
|
|
234
|
-
* pre-loaded data without a runtime dependency.
|
|
235
|
-
*/
|
|
236
|
-
type DictionaryMeta = {
|
|
237
|
-
label: string;
|
|
238
|
-
category: DenyListCategory;
|
|
239
|
-
country: string | null;
|
|
240
|
-
};
|
|
241
|
-
/**
|
|
242
|
-
* Caller-supplied exact terms for deny-list matching.
|
|
243
|
-
* These entries are merged with the published deny-list
|
|
244
|
-
* dictionaries when `enableDenyList` is enabled.
|
|
245
|
-
*/
|
|
246
|
-
type CustomDenyListEntry = {
|
|
247
|
-
value: string;
|
|
248
|
-
label: string;
|
|
249
|
-
variants?: readonly string[];
|
|
250
|
-
};
|
|
251
|
-
/**
|
|
252
|
-
* Caller-supplied regex detector. The pattern is passed
|
|
253
|
-
* to the underlying text-search regex engine, so use its
|
|
254
|
-
* supported regex syntax. Inline flags such as `(?i)` are
|
|
255
|
-
* accepted when supported by that engine.
|
|
256
|
-
*/
|
|
257
|
-
type CustomRegexPattern = {
|
|
258
|
-
pattern: string;
|
|
259
|
-
label: string;
|
|
260
|
-
score?: number;
|
|
261
|
-
preparedArtifactPolicy?: "include" | "omit";
|
|
262
|
-
};
|
|
263
|
-
/**
|
|
264
|
-
* Pre-loaded dictionary data for dependency injection.
|
|
265
|
-
* Consumers that want name/city/deny-list detection
|
|
266
|
-
* load dictionaries themselves (e.g. from the
|
|
267
|
-
* anonymize-data package) and pass them here; the
|
|
268
|
-
* anonymize package has zero cross-package imports.
|
|
269
|
-
*
|
|
270
|
-
* All fields are optional. When a field is absent,
|
|
271
|
-
* the corresponding detection path is skipped (same
|
|
272
|
-
* behavior as when no dictionaries are available).
|
|
273
|
-
*/
|
|
274
|
-
type Dictionaries = {
|
|
275
|
-
/**
|
|
276
|
-
* First names per language code (e.g., "cs", "de").
|
|
277
|
-
* Merged with legacy config names at init time.
|
|
278
|
-
*/
|
|
279
|
-
firstNames?: Readonly<Record<string, readonly string[]>>;
|
|
280
|
-
/**
|
|
281
|
-
* Surnames per language code.
|
|
282
|
-
* Merged with legacy config names at init time.
|
|
283
|
-
*/
|
|
284
|
-
surnames?: Readonly<Record<string, readonly string[]>>;
|
|
285
|
-
/**
|
|
286
|
-
* Non-Western name tokens per locale code
|
|
287
|
-
* (e.g., "in", "ar", "ja-latn", "ko", "zh-latn",
|
|
288
|
-
* "th", "vi", "fil", "id"). Merged with bundled
|
|
289
|
-
* names-nw-*.json data at init time.
|
|
290
|
-
*/
|
|
291
|
-
nonWesternNames?: Readonly<Record<string, readonly string[]>>;
|
|
292
|
-
/**
|
|
293
|
-
* Pre-loaded deny-list dictionaries keyed by
|
|
294
|
-
* dictionary ID (e.g., "courts/CZ", "banks/DE").
|
|
295
|
-
* Each value is the array of terms for that
|
|
296
|
-
* dictionary.
|
|
297
|
-
*/
|
|
298
|
-
denyList?: Readonly<Record<string, readonly string[]>>;
|
|
299
|
-
/**
|
|
300
|
-
* Metadata per dictionary ID. Required when
|
|
301
|
-
* `denyList` is provided so the pipeline knows
|
|
302
|
-
* labels, categories, and country filters.
|
|
303
|
-
*/
|
|
304
|
-
denyListMeta?: Readonly<Record<string, DictionaryMeta>>;
|
|
305
|
-
/**
|
|
306
|
-
* Pre-loaded city names, already merged across
|
|
307
|
-
* all desired countries.
|
|
308
|
-
*
|
|
309
|
-
* Prefer `citiesByCountry` when callers also pass
|
|
310
|
-
* `denyListCountries` / `denyListRegions`; merged
|
|
311
|
-
* city arrays cannot be scoped after injection.
|
|
312
|
-
*/
|
|
313
|
-
cities?: readonly string[];
|
|
314
|
-
/**
|
|
315
|
-
* Pre-loaded city names keyed by ISO 3166-1 alpha-2
|
|
316
|
-
* country code. When provided, the deny-list builder
|
|
317
|
-
* applies `denyListCountries` / `denyListRegions`
|
|
318
|
-
* before adding city patterns to the search automaton.
|
|
319
|
-
*/
|
|
320
|
-
citiesByCountry?: Readonly<Record<string, readonly string[]>>;
|
|
321
|
-
};
|
|
322
|
-
type PipelineConfig = {
|
|
323
|
-
threshold: number;
|
|
324
|
-
enableTriggerPhrases: boolean;
|
|
325
|
-
enableRegex: boolean;
|
|
326
|
-
/**
|
|
327
|
-
* Expected content language codes. When present, these
|
|
328
|
-
* derive default dictionary scopes for name corpus and
|
|
329
|
-
* deny-list matching unless the lower-level scope fields
|
|
330
|
-
* below are set explicitly.
|
|
331
|
-
*/
|
|
332
|
-
languages?: string[];
|
|
333
|
-
/**
|
|
334
|
-
* Convenience form for single-language documents. Ignored
|
|
335
|
-
* when `languages` is also provided.
|
|
336
|
-
*/
|
|
337
|
-
language?: string;
|
|
338
|
-
/**
|
|
339
|
-
* Enables legal-form organization detection.
|
|
340
|
-
* Required for typed callers; legacy untyped
|
|
341
|
-
* callers that omit this field are treated as
|
|
342
|
-
* enabled at runtime for backward compatibility.
|
|
343
|
-
*/
|
|
344
|
-
enableLegalForms: boolean;
|
|
345
|
-
/**
|
|
346
|
-
* Enables first-name/surname/title corpus matching.
|
|
347
|
-
* When deny-list mode is enabled, this also controls
|
|
348
|
-
* whether name-corpus entries are injected into the
|
|
349
|
-
* deny-list search automaton.
|
|
350
|
-
*/
|
|
351
|
-
enableNameCorpus: boolean;
|
|
352
|
-
/**
|
|
353
|
-
* Optional language scope for first-name/surname
|
|
354
|
-
* dictionaries, using the keys present in
|
|
355
|
-
* `dictionaries.firstNames` / `dictionaries.surnames`
|
|
356
|
-
* (for example `["en", "de"]`). When omitted, all
|
|
357
|
-
* injected name languages are used for backward
|
|
358
|
-
* compatibility.
|
|
359
|
-
*/
|
|
360
|
-
nameCorpusLanguages?: string[];
|
|
361
|
-
enableDenyList: boolean;
|
|
362
|
-
denyListCountries?: string[];
|
|
363
|
-
denyListRegions?: string[];
|
|
364
|
-
denyListExcludeCategories?: string[];
|
|
365
|
-
/**
|
|
366
|
-
* Caller-owned exact terms to match through the
|
|
367
|
-
* deny-list layer. Requires `enableDenyList: true`.
|
|
368
|
-
*/
|
|
369
|
-
customDenyList?: readonly CustomDenyListEntry[];
|
|
370
|
-
/**
|
|
371
|
-
* Caller-owned regex detectors. Requires
|
|
372
|
-
* `enableRegex: true`.
|
|
373
|
-
*/
|
|
374
|
-
customRegexes?: readonly CustomRegexPattern[];
|
|
375
|
-
enableGazetteer: boolean;
|
|
376
|
-
/**
|
|
377
|
-
* Detect country names (ISO 3166-1 names, curated
|
|
378
|
-
* aliases, alpha-3 codes). Defaults to true. Names
|
|
379
|
-
* span all manifest languages plus widely-used
|
|
380
|
-
* additions (Dutch, Russian, Chinese, Arabic, etc.).
|
|
381
|
-
*/
|
|
382
|
-
enableCountries?: boolean;
|
|
383
|
-
enableNer: boolean;
|
|
384
|
-
enableConfidenceBoost: boolean;
|
|
385
|
-
enableCoreference: boolean;
|
|
386
|
-
enableZoneClassification?: boolean;
|
|
387
|
-
enableHotwordRules?: boolean;
|
|
388
|
-
/**
|
|
389
|
-
* Requested output labels. An empty array means
|
|
390
|
-
* "do not filter by label" for deterministic
|
|
391
|
-
* detectors; NER falls back to DEFAULT_ENTITY_LABELS.
|
|
392
|
-
*/
|
|
393
|
-
labels: string[];
|
|
394
|
-
workspaceId: string;
|
|
395
|
-
/**
|
|
396
|
-
* Pre-loaded dictionary data for name, deny-list,
|
|
397
|
-
* and city detection. When omitted, dictionary-based
|
|
398
|
-
* detection paths are skipped. Consumers load from
|
|
399
|
-
* the anonymize-data package and pass the data here.
|
|
400
|
-
*/
|
|
401
|
-
dictionaries?: Dictionaries;
|
|
402
|
-
};
|
|
403
|
-
//#endregion
|
|
404
|
-
//#region src/detectors/regex.d.ts
|
|
405
|
-
type RegexMeta = {
|
|
406
|
-
label: string;
|
|
407
|
-
score: number;
|
|
408
|
-
sourceDetail?: Entity["sourceDetail"];
|
|
409
|
-
minByteLength?: number; /** Post-match stdnum validator for confirmation. */
|
|
410
|
-
validator?: Validator;
|
|
411
|
-
validatorId?: string; /** Extract the identifier portion when context is part of the regex span. */
|
|
412
|
-
validatorInput?: (text: string) => string;
|
|
413
|
-
validatorInputKind?: "digits-only" | "crypto-wallet-candidate";
|
|
414
|
-
};
|
|
415
|
-
/** Flat pattern array for text-search. */
|
|
416
|
-
declare const REGEX_PATTERNS: readonly string[];
|
|
417
|
-
/** Parallel metadata. Index = pattern index. */
|
|
418
|
-
declare const REGEX_META: readonly RegexMeta[];
|
|
419
|
-
type DateMonthData = Record<string, string[]>;
|
|
420
|
-
type YearWordData = Record<string, string[]>;
|
|
421
|
-
/**
|
|
422
|
-
* Get dynamically built date patterns from
|
|
423
|
-
* date-months.json. Returns a cached promise; the JSON
|
|
424
|
-
* is loaded only once.
|
|
425
|
-
*/
|
|
426
|
-
declare const getDatePatterns: (languages?: readonly string[]) => Promise<string[]>;
|
|
427
|
-
/** Date pattern metadata (all are score 1 dates). */
|
|
428
|
-
declare const DATE_PATTERN_META: Readonly<RegexMeta>;
|
|
429
|
-
type MonetaryData = {
|
|
430
|
-
currencies: {
|
|
431
|
-
codes: string[];
|
|
432
|
-
symbols: string[];
|
|
433
|
-
local_names: string[];
|
|
434
|
-
};
|
|
435
|
-
amount_words: {
|
|
436
|
-
written_amount_patterns: Array<{
|
|
437
|
-
keywords: string[];
|
|
438
|
-
}>;
|
|
439
|
-
magnitude_suffixes: Array<{
|
|
440
|
-
words: string[];
|
|
441
|
-
abbreviations_case_insensitive: string[];
|
|
442
|
-
abbreviations_case_sensitive: string[];
|
|
443
|
-
}>;
|
|
444
|
-
share_quantity_terms: Array<{
|
|
445
|
-
modifiers: string[];
|
|
446
|
-
nouns: string[];
|
|
447
|
-
}>;
|
|
448
|
-
};
|
|
449
|
-
};
|
|
450
|
-
/**
|
|
451
|
-
* Get dynamically built monetary amount patterns from
|
|
452
|
-
* currencies.json. Returns a cached promise; the JSON
|
|
453
|
-
* is loaded only once.
|
|
454
|
-
*/
|
|
455
|
-
declare const getCurrencyPatterns: () => Promise<string[]>;
|
|
456
|
-
/** Currency pattern metadata (score 0.9). */
|
|
457
|
-
declare const CURRENCY_PATTERN_META: Readonly<RegexMeta>;
|
|
458
|
-
/**
|
|
459
|
-
* Process regex matches from the unified search.
|
|
460
|
-
* Receives all matches; filters to the regex slice
|
|
461
|
-
* via sliceStart/sliceEnd. Local index into META is
|
|
462
|
-
* match.pattern - sliceStart.
|
|
463
|
-
*
|
|
464
|
-
* For stdnum-derived patterns (those with a validator
|
|
465
|
-
* in META), the matched text is passed through the
|
|
466
|
-
* validator's validate() method. If validation fails,
|
|
467
|
-
* the match is discarded as a false positive.
|
|
468
|
-
*/
|
|
469
|
-
declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, meta_: readonly RegexMeta[]) => Entity[];
|
|
470
|
-
//#endregion
|
|
471
|
-
//#region src/detectors/deny-list.d.ts
|
|
472
|
-
type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries" | "enableCountries">;
|
|
473
|
-
type DenyListFilterData = {
|
|
474
|
-
stopwords: string[];
|
|
475
|
-
allowList: string[];
|
|
476
|
-
personStopwords: string[];
|
|
477
|
-
personTrailingNouns: string[];
|
|
478
|
-
addressStopwords: string[];
|
|
479
|
-
addressJurisdictionPrefixes: string[];
|
|
480
|
-
streetTypes: string[];
|
|
481
|
-
addressComponentTerms: string[];
|
|
482
|
-
ambiguousStreetTypeTerms: string[];
|
|
483
|
-
firstNames: string[];
|
|
484
|
-
genericRoles: string[];
|
|
485
|
-
numberAbbrevPrefixes: string[];
|
|
486
|
-
sentenceStarters: string[];
|
|
487
|
-
trailingAddressWordExclusions: string[];
|
|
488
|
-
documentHeadingWords: string[];
|
|
489
|
-
documentHeadingOrdinalMarkers: string[];
|
|
490
|
-
definedTermCues: string[];
|
|
491
|
-
signingPlaceGuards: DenyListSigningPlaceGuardData[];
|
|
492
|
-
};
|
|
493
|
-
type DenyListSigningPlaceGuardData = {
|
|
494
|
-
prefixPhrases: string[];
|
|
495
|
-
suffixPhrases: string[];
|
|
496
|
-
};
|
|
497
|
-
/**
|
|
498
|
-
* Source tag for each pattern in the automaton.
|
|
499
|
-
* "deny-list" = standard deny list entry
|
|
500
|
-
* "city" = city dictionary entry
|
|
501
|
-
* "custom-deny-list" = caller-owned exact term
|
|
502
|
-
* "first-name" = name corpus first name
|
|
503
|
-
* "surname" = name corpus surname
|
|
504
|
-
* "title" = academic/professional title
|
|
505
|
-
*/
|
|
506
|
-
type PatternSource = "deny-list" | "city" | "custom-deny-list" | "first-name" | "surname" | "title";
|
|
507
|
-
type PatternLabels = string | string[];
|
|
508
|
-
type PatternSources = PatternSource | PatternSource[];
|
|
509
|
-
/**
|
|
510
|
-
* Pre-built deny list data. Constructed once by
|
|
511
|
-
* `buildDenyList`, reused across `processDenyListMatches`
|
|
512
|
-
* calls. Contains PatternEntry[] for the unified builder
|
|
513
|
-
* plus parallel label/source arrays for post-processing.
|
|
514
|
-
*/
|
|
515
|
-
type DenyListData = {
|
|
516
|
-
/**
|
|
517
|
-
* Maps pattern index → entity labels (plural).
|
|
518
|
-
* Same pattern can have multiple labels when it
|
|
519
|
-
* appears in multiple dictionaries (e.g., "Denver"
|
|
520
|
-
* is both a person name and a city name).
|
|
521
|
-
*/
|
|
522
|
-
labels: PatternLabels[]; /** Maps pattern index → labels contributed by custom entries. */
|
|
523
|
-
customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
|
|
524
|
-
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
525
|
-
sources: PatternSources[];
|
|
526
|
-
filters: DenyListFilterData;
|
|
527
|
-
};
|
|
528
|
-
/**
|
|
529
|
-
* Resolve which dictionaries to load based on country
|
|
530
|
-
* and category filters, then build the deny list data.
|
|
531
|
-
* The returned data provides PatternEntry[] for the
|
|
532
|
-
* unified builder and parallel arrays for
|
|
533
|
-
* post-processing.
|
|
534
|
-
*
|
|
535
|
-
* Dictionary data is injected via `config.dictionaries`.
|
|
536
|
-
* Returns null if no dictionaries are provided.
|
|
537
|
-
*/
|
|
538
|
-
declare const buildDenyList: (config: DenyListConfig, ctx?: PipelineContext) => Promise<DenyListData | null>;
|
|
539
|
-
/**
|
|
540
|
-
* Ensure all deny-list support data (stopwords, allow
|
|
541
|
-
* list, person stopwords, generic roles) is loaded on
|
|
542
|
-
* the given context. Call this before
|
|
543
|
-
* processDenyListMatches / filterFalsePositives when
|
|
544
|
-
* the search instance was built on a different context
|
|
545
|
-
* (e.g. cachedSearch).
|
|
546
|
-
*/
|
|
547
|
-
declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Dictionaries, nameCorpusLanguages?: readonly string[]) => Promise<void>;
|
|
548
|
-
/**
|
|
549
|
-
* Process deny list matches from the unified search.
|
|
550
|
-
* Receives all matches; filters to the deny list slice
|
|
551
|
-
* via sliceStart/sliceEnd. Local index into data.labels,
|
|
552
|
-
* data.originals, data.sources is match.pattern - sliceStart.
|
|
553
|
-
*
|
|
554
|
-
* Two-pass approach to reduce false positives:
|
|
555
|
-
* 1. Collect all matches (case-insensitive,
|
|
556
|
-
* whole-word via Rust automaton)
|
|
557
|
-
* 2. Require uppercase start in source text
|
|
558
|
-
* 3. For person names, require at least one
|
|
559
|
-
* mid-sentence occurrence to prove proper noun
|
|
560
|
-
* 4. Return all occurrences of validated terms
|
|
561
|
-
*/
|
|
562
|
-
declare const processDenyListMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: DenyListData, ctx?: PipelineContext) => Entity[];
|
|
563
|
-
//#endregion
|
|
564
|
-
//#region src/detectors/address-seeds.d.ts
|
|
565
|
-
type AddressSeedData = {
|
|
566
|
-
boundary_words: string[];
|
|
567
|
-
br_cep_cue_words: string[];
|
|
568
|
-
unit_abbreviations: string[];
|
|
569
|
-
};
|
|
570
|
-
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
571
|
-
/**
|
|
572
|
-
* Process address seeds from the unified search.
|
|
573
|
-
* Receives all matches; filters to the street types
|
|
574
|
-
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
575
|
-
* existingEntities for seed collection, clustering,
|
|
576
|
-
* expansion, and scoring.
|
|
577
|
-
*
|
|
578
|
-
* Runs as a post-processor after all other detectors,
|
|
579
|
-
* using their output as seed sources.
|
|
580
|
-
*/
|
|
581
|
-
declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
|
|
582
|
-
//#endregion
|
|
583
|
-
//#region src/detectors/countries.d.ts
|
|
584
|
-
/**
|
|
585
|
-
* Pre-built country patterns + parallel label/source
|
|
586
|
-
* metadata. Constructed once and reused across pipeline
|
|
587
|
-
* runs.
|
|
588
|
-
*/
|
|
589
|
-
type CountryData = {
|
|
590
|
-
/** Maps local pattern index to entity label. Always "country". */labels: string[];
|
|
591
|
-
/**
|
|
592
|
-
* Maps local pattern index to the alpha-2 ISO code the
|
|
593
|
-
* pattern resolves to. Used for downstream coreference /
|
|
594
|
-
* placeholder grouping.
|
|
595
|
-
*/
|
|
596
|
-
isoCodes: string[]; /** Maps local pattern index to pattern variant kind. */
|
|
597
|
-
variants: CountryVariant[];
|
|
598
|
-
};
|
|
599
|
-
type CountryVariant = "name" | "alias" | "alpha3" | "alpha2";
|
|
600
|
-
//#endregion
|
|
601
|
-
//#region src/filters/confidence-boost.d.ts
|
|
602
|
-
/**
|
|
603
|
-
* Boost confidence of near-miss NER entities that appear
|
|
604
|
-
* near high-confidence detections (regex, trigger phrase).
|
|
605
|
-
*
|
|
606
|
-
* If an NER entity scored between (threshold - 0.15) and
|
|
607
|
-
* threshold, count how many confirmed entities exist within
|
|
608
|
-
* a 150-char window. Add +0.05 per co-located entity.
|
|
609
|
-
* If the boosted score crosses the threshold, include it.
|
|
610
|
-
*
|
|
611
|
-
* Only mutates score on near-miss entities; high-confidence
|
|
612
|
-
* entities pass through unchanged.
|
|
613
|
-
*/
|
|
614
|
-
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
615
|
-
type AddressContextData = {
|
|
616
|
-
address_prepositions: string[];
|
|
617
|
-
temporal_prepositions: string[];
|
|
618
|
-
street_abbreviations: string[];
|
|
619
|
-
bare_house_stopwords: string[];
|
|
620
|
-
};
|
|
621
|
-
//#endregion
|
|
622
|
-
//#region src/build-unified-search.d.ts
|
|
623
16
|
type PatternSlice = {
|
|
624
17
|
start: number;
|
|
625
18
|
end: number;
|
|
@@ -657,6 +50,10 @@ type NativeRegexMatchMeta = {
|
|
|
657
50
|
validator_input?: string;
|
|
658
51
|
min_byte_length?: number;
|
|
659
52
|
};
|
|
53
|
+
type NativeSigningPlaceGuardData = {
|
|
54
|
+
prefix_phrases: string[];
|
|
55
|
+
suffix_phrases: string[];
|
|
56
|
+
};
|
|
660
57
|
type NativeDenyListFilterData = {
|
|
661
58
|
stopwords: string[];
|
|
662
59
|
allow_list: string[];
|
|
@@ -677,10 +74,6 @@ type NativeDenyListFilterData = {
|
|
|
677
74
|
defined_term_cues: string[];
|
|
678
75
|
signing_place_guards: NativeSigningPlaceGuardData[];
|
|
679
76
|
};
|
|
680
|
-
type NativeSigningPlaceGuardData = {
|
|
681
|
-
prefix_phrases: string[];
|
|
682
|
-
suffix_phrases: string[];
|
|
683
|
-
};
|
|
684
77
|
type NativeDenyListMatchData = {
|
|
685
78
|
labels?: string[][];
|
|
686
79
|
label_table?: string[];
|
|
@@ -767,13 +160,44 @@ type NativeLegalFormData = {
|
|
|
767
160
|
company_suffix_words: string[];
|
|
768
161
|
comma_gated_direct_prefixes: string[];
|
|
769
162
|
};
|
|
163
|
+
type NativeDateMonthData = Record<string, string[]>;
|
|
164
|
+
type NativeYearWordData = Record<string, string[]>;
|
|
770
165
|
type NativeDateData = {
|
|
771
|
-
month_names_by_language:
|
|
772
|
-
year_words_by_language:
|
|
166
|
+
month_names_by_language: NativeDateMonthData;
|
|
167
|
+
year_words_by_language: NativeYearWordData;
|
|
168
|
+
};
|
|
169
|
+
type NativeMonetaryData = {
|
|
170
|
+
currencies: {
|
|
171
|
+
codes: string[];
|
|
172
|
+
symbols: string[];
|
|
173
|
+
local_names: string[];
|
|
174
|
+
};
|
|
175
|
+
amount_words: {
|
|
176
|
+
written_amount_patterns: Array<{
|
|
177
|
+
keywords: string[];
|
|
178
|
+
}>;
|
|
179
|
+
magnitude_suffixes: Array<{
|
|
180
|
+
words: string[];
|
|
181
|
+
abbreviations_case_insensitive: string[];
|
|
182
|
+
abbreviations_case_sensitive: string[];
|
|
183
|
+
}>;
|
|
184
|
+
share_quantity_terms: Array<{
|
|
185
|
+
modifiers: string[];
|
|
186
|
+
nouns: string[];
|
|
187
|
+
}>;
|
|
188
|
+
};
|
|
189
|
+
};
|
|
190
|
+
type NativeAddressSeedData = {
|
|
191
|
+
boundary_words: string[];
|
|
192
|
+
br_cep_cue_words: string[];
|
|
193
|
+
unit_abbreviations: string[];
|
|
194
|
+
};
|
|
195
|
+
type NativeAddressContextData = {
|
|
196
|
+
address_prepositions: string[];
|
|
197
|
+
temporal_prepositions: string[];
|
|
198
|
+
street_abbreviations: string[];
|
|
199
|
+
bare_house_stopwords: string[];
|
|
773
200
|
};
|
|
774
|
-
type NativeMonetaryData = MonetaryData;
|
|
775
|
-
type NativeAddressSeedData = AddressSeedData;
|
|
776
|
-
type NativeAddressContextData = AddressContextData;
|
|
777
201
|
type NativeCoreferencePatternData = {
|
|
778
202
|
pattern: string;
|
|
779
203
|
flags: string;
|
|
@@ -816,6 +240,11 @@ type NativeZoneData = {
|
|
|
816
240
|
section_heading_patterns: NativeZonePatternData[];
|
|
817
241
|
signing_clauses: NativeZoneSigningClauseData[];
|
|
818
242
|
};
|
|
243
|
+
type NativeCountryData = {
|
|
244
|
+
labels: string[];
|
|
245
|
+
isoCodes: string[];
|
|
246
|
+
variants: Array<"name" | "alias" | "alpha3" | "alpha2">;
|
|
247
|
+
};
|
|
819
248
|
type NativeGazetteerData = {
|
|
820
249
|
labels: string[];
|
|
821
250
|
is_fuzzy: boolean[];
|
|
@@ -867,7 +296,7 @@ type NativePreparedSearchConfig = {
|
|
|
867
296
|
deny_list_data?: NativeDenyListMatchData;
|
|
868
297
|
false_positive_filters?: NativeDenyListFilterData;
|
|
869
298
|
gazetteer_data?: NativeGazetteerData;
|
|
870
|
-
country_data?:
|
|
299
|
+
country_data?: NativeCountryData;
|
|
871
300
|
hotword_data?: NativeHotwordRuleData;
|
|
872
301
|
trigger_data?: NativeTriggerData;
|
|
873
302
|
legal_form_data?: NativeLegalFormData;
|
|
@@ -881,188 +310,86 @@ type NativePreparedSearchConfig = {
|
|
|
881
310
|
date_data?: NativeDateData;
|
|
882
311
|
monetary_data?: NativeMonetaryData;
|
|
883
312
|
};
|
|
884
|
-
type GazetteerData = {
|
|
885
|
-
/** Maps local pattern index to entry label. */labels: string[];
|
|
886
|
-
/**
|
|
887
|
-
* Whether each pattern is fuzzy (distance > 0).
|
|
888
|
-
* Used by the post-processor to assign scores.
|
|
889
|
-
*/
|
|
890
|
-
isFuzzy: boolean[];
|
|
891
|
-
};
|
|
892
|
-
type UnifiedSearchInstance = {
|
|
893
|
-
/** Regex + triggers + legal-forms. */tsRegex: TextSearch; /** Caller-owned custom regexes, isolated for overlap preservation. */
|
|
894
|
-
tsCustomRegex: TextSearch; /** Deny-list + street-types + gazetteer. */
|
|
895
|
-
tsLiterals: TextSearch;
|
|
896
|
-
slices: {
|
|
897
|
-
regex: PatternSlice;
|
|
898
|
-
customRegex: PatternSlice;
|
|
899
|
-
legalForms: PatternSlice;
|
|
900
|
-
triggers: PatternSlice;
|
|
901
|
-
denyList: PatternSlice;
|
|
902
|
-
streetTypes: PatternSlice;
|
|
903
|
-
gazetteer: PatternSlice;
|
|
904
|
-
countries: PatternSlice;
|
|
905
|
-
};
|
|
906
|
-
regexMeta: readonly RegexMeta[];
|
|
907
|
-
customRegexMeta: readonly RegexMeta[];
|
|
908
|
-
triggerRules: readonly TriggerRule[];
|
|
909
|
-
denyListData: DenyListData | null;
|
|
910
|
-
gazetteerData: GazetteerData | null;
|
|
911
|
-
countryData: CountryData | null;
|
|
912
|
-
nativeStaticConfig: NativePreparedSearchConfig;
|
|
913
|
-
};
|
|
914
|
-
declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: GazetteerEntry[], ctx?: PipelineContext) => Promise<UnifiedSearchInstance>;
|
|
915
313
|
//#endregion
|
|
916
|
-
//#region src/
|
|
917
|
-
/**
|
|
918
|
-
* Build a stable cache key for an entity that survives
|
|
919
|
-
* shallow copies (spread). Uses position + label so the
|
|
920
|
-
* key is identical for the original object and any
|
|
921
|
-
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
922
|
-
*
|
|
923
|
-
* @deprecated No longer used internally: coref alias
|
|
924
|
-
* links travel on the entities themselves
|
|
925
|
-
* (`corefSourceText`). Kept for API compatibility.
|
|
926
|
-
*/
|
|
927
|
-
declare const corefKey: (e: Entity) => string;
|
|
314
|
+
//#region src/types.d.ts
|
|
928
315
|
/**
|
|
929
|
-
*
|
|
930
|
-
* definition extraction.
|
|
316
|
+
* Fields shared by every entity span in the source text.
|
|
931
317
|
*/
|
|
932
|
-
type
|
|
933
|
-
|
|
318
|
+
type EntityBase = {
|
|
319
|
+
start: number;
|
|
320
|
+
end: number;
|
|
321
|
+
label: string;
|
|
322
|
+
text: string;
|
|
323
|
+
score: number;
|
|
324
|
+
sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
|
|
934
325
|
};
|
|
935
326
|
/**
|
|
936
|
-
*
|
|
937
|
-
*
|
|
938
|
-
* detectNameCorpus and deny-list AC integration.
|
|
327
|
+
* A PII entity span found by a primary detection layer
|
|
328
|
+
* (regex, NER, legal forms, deny list, ...).
|
|
939
329
|
*/
|
|
940
|
-
type
|
|
941
|
-
|
|
942
|
-
surnames: ReadonlySet<string>;
|
|
943
|
-
titleTokens: ReadonlySet<string>;
|
|
944
|
-
/** Abbreviation-style titles whose trailing dot is
|
|
945
|
-
* part of the title, not a sentence boundary.
|
|
946
|
-
* Contains the lowercase, dot-stripped form
|
|
947
|
-
* (e.g., "dr", "smt", "atty"). */
|
|
948
|
-
titleAbbreviations: ReadonlySet<string>;
|
|
949
|
-
excludedWords: ReadonlySet<string>;
|
|
950
|
-
/** Lowercased common English words. A name chain whose
|
|
951
|
-
* every token is a common word (e.g. "Loan Documents",
|
|
952
|
-
* where "Loan" coincides with a Vietnamese given name)
|
|
953
|
-
* is treated as a common-word phrase, not a person. */
|
|
954
|
-
commonWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
|
|
955
|
-
nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
|
|
956
|
-
excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
|
|
957
|
-
firstNamesList: readonly string[];
|
|
958
|
-
surnamesList: readonly string[];
|
|
959
|
-
titlesList: readonly string[];
|
|
960
|
-
excludedList: readonly string[];
|
|
961
|
-
nonWesternNamesList: readonly string[];
|
|
962
|
-
excludedAllCapsList: readonly string[];
|
|
330
|
+
type DetectedEntity = EntityBase & {
|
|
331
|
+
source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
|
|
963
332
|
};
|
|
964
333
|
/**
|
|
965
|
-
*
|
|
966
|
-
*
|
|
967
|
-
*
|
|
968
|
-
* concurrent pipelines with different configs and
|
|
969
|
-
* simplifies testing.
|
|
334
|
+
* An alias mention of a previously detected entity: a
|
|
335
|
+
* defined term ("the Seller") or a propagated bare
|
|
336
|
+
* mention ("Acme" after "Acme Corp.").
|
|
970
337
|
*
|
|
971
|
-
*
|
|
972
|
-
*
|
|
338
|
+
* `corefSourceText` is required by construction, so an
|
|
339
|
+
* alias cannot exist without the link back to its source
|
|
340
|
+
* entity. Placeholder numbering reads it to give the
|
|
341
|
+
* alias the same placeholder as the source. The link
|
|
342
|
+
* travels with the entity instead of living in a
|
|
343
|
+
* side-channel map that a producer could forget to
|
|
344
|
+
* write — or that a later pass could clear.
|
|
973
345
|
*/
|
|
974
|
-
type
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
searchPromise: Promise<UnifiedSearchInstance> | null;
|
|
978
|
-
nativePipelinePackage: Uint8Array | null;
|
|
979
|
-
nativePipelinePackageKey: string;
|
|
980
|
-
nativePipelinePackagePromise: Promise<Uint8Array> | null;
|
|
981
|
-
nameCorpus: NameCorpusData | null;
|
|
982
|
-
nameCorpusKey: string;
|
|
983
|
-
nameCorpusPromise: Promise<void> | null;
|
|
984
|
-
stopwords: ReadonlySet<string> | null;
|
|
985
|
-
stopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
986
|
-
allowList: ReadonlySet<string> | null;
|
|
987
|
-
allowListPromise: Promise<ReadonlySet<string>> | null;
|
|
988
|
-
personStopwords: ReadonlySet<string> | null;
|
|
989
|
-
personStopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
990
|
-
definedTermHeads: ReadonlySet<string> | null;
|
|
991
|
-
definedTermHeadsPromise: Promise<ReadonlySet<string>> | null;
|
|
992
|
-
addressStopwords: ReadonlySet<string> | null;
|
|
993
|
-
addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
|
|
994
|
-
firstNameExclusions: ReadonlySet<string> | null;
|
|
995
|
-
firstNameExclusionCorpusLen: number;
|
|
996
|
-
genericRoles: ReadonlySet<string> | null;
|
|
997
|
-
genericRolesPromise: Promise<ReadonlySet<string>> | null;
|
|
998
|
-
corefPatterns: DefinitionPattern[] | null;
|
|
999
|
-
corefPatternsKey: string;
|
|
1000
|
-
corefPatternsPromise: Promise<DefinitionPattern[]> | null;
|
|
1001
|
-
corefLoadAttempted: boolean;
|
|
1002
|
-
roleStopSet: ReadonlySet<string> | null;
|
|
1003
|
-
roleStopSetPromise: Promise<ReadonlySet<string>> | null;
|
|
1004
|
-
zoneHeadingPatterns: RegExp[] | null;
|
|
1005
|
-
zoneSigningPatterns: RegExp[] | null;
|
|
1006
|
-
zoneInitPromise: Promise<void> | null;
|
|
1007
|
-
};
|
|
1008
|
-
/** Create a fresh, empty pipeline context. */
|
|
1009
|
-
declare const createPipelineContext: () => PipelineContext;
|
|
1010
|
-
//#endregion
|
|
1011
|
-
//#region src/pipeline.d.ts
|
|
1012
|
-
/** Strip leading/trailing whitespace and punctuation. */
|
|
1013
|
-
declare const sanitizeEntities: (entities: Entity[]) => Entity[];
|
|
1014
|
-
declare const mergeAndDedup: (...layers: Entity[][]) => Entity[];
|
|
1015
|
-
type NerInferenceFn = (fullText: string, labels: string[], threshold: number, signal?: AbortSignal) => Promise<Entity[]>;
|
|
1016
|
-
type PipelineSearchOptions = {
|
|
1017
|
-
config: PipelineConfig;
|
|
1018
|
-
gazetteerEntries?: GazetteerEntry[];
|
|
1019
|
-
context?: PipelineContext;
|
|
346
|
+
type CorefAliasEntity = EntityBase & {
|
|
347
|
+
source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
|
|
348
|
+
corefSourceText: string;
|
|
1020
349
|
};
|
|
1021
350
|
/**
|
|
1022
|
-
*
|
|
1023
|
-
*
|
|
1024
|
-
* `runPipeline` to reuse the prepared automata without
|
|
1025
|
-
* passing `cachedSearch` around manually.
|
|
351
|
+
* A detected PII entity span in the source text.
|
|
352
|
+
* Every detection layer produces these.
|
|
1026
353
|
*/
|
|
1027
|
-
|
|
1028
|
-
config,
|
|
1029
|
-
gazetteerEntries,
|
|
1030
|
-
context
|
|
1031
|
-
}: PipelineSearchOptions) => Promise<UnifiedSearchInstance>;
|
|
354
|
+
type Entity = DetectedEntity | CorefAliasEntity;
|
|
1032
355
|
/**
|
|
1033
|
-
*
|
|
1034
|
-
*
|
|
1035
|
-
* @property cachedSearch Pre-built search instance.
|
|
1036
|
-
* When provided, `config` and `gazetteerEntries`
|
|
1037
|
-
* are not used for building; the caller must
|
|
1038
|
-
* ensure the instance matches both parameters.
|
|
356
|
+
* Entity after human review. Extends the base Entity
|
|
357
|
+
* with a review decision.
|
|
1039
358
|
*/
|
|
1040
|
-
type
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
359
|
+
type ReviewDecision = "confirmed" | "rejected" | "relabeled";
|
|
360
|
+
type ReviewedEntity = Entity & {
|
|
361
|
+
decision?: ReviewDecision;
|
|
362
|
+
originalLabel?: string;
|
|
363
|
+
};
|
|
364
|
+
/** Per-label operator selection. Key is the entity label. */
|
|
365
|
+
type OperatorConfig = {
|
|
366
|
+
/** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
|
|
367
|
+
redactString: string;
|
|
368
|
+
};
|
|
369
|
+
/** Whether an operator produces a reversible redaction entry. */
|
|
370
|
+
type OperatorReversibility = "reversible" | "irreversible";
|
|
371
|
+
type AnonymisationOperator = {
|
|
372
|
+
type: OperatorType;
|
|
373
|
+
reversibility: OperatorReversibility;
|
|
374
|
+
/**
|
|
375
|
+
* Apply the operator to a single entity occurrence.
|
|
376
|
+
* Returns the replacement string to embed in the document.
|
|
377
|
+
*/
|
|
378
|
+
apply: (text: string, label: string, placeholder: string, redactString: string) => string;
|
|
1049
379
|
};
|
|
1050
380
|
/**
|
|
1051
|
-
*
|
|
1052
|
-
*
|
|
1053
|
-
* Two TextSearch instances scan the text (regex +
|
|
1054
|
-
* literals). Results are dispatched to each
|
|
1055
|
-
* detector's post-processor by pattern index range.
|
|
1056
|
-
*
|
|
1057
|
-
* Pass an AbortSignal to cancel the pipeline between
|
|
1058
|
-
* stages. Throws a DOMException with name "AbortError"
|
|
1059
|
-
* when cancelled.
|
|
1060
|
-
*
|
|
1061
|
-
* Pass an optional `context` to isolate cached state
|
|
1062
|
-
* from other pipeline runs. If omitted, a module-level
|
|
1063
|
-
* default context is used (backward compatible).
|
|
381
|
+
* Redacted document output with stable entity mapping.
|
|
1064
382
|
*/
|
|
1065
|
-
|
|
383
|
+
type RedactionResult = {
|
|
384
|
+
redactedText: string;
|
|
385
|
+
/**
|
|
386
|
+
* Maps placeholder to original text. Only populated for
|
|
387
|
+
* reversible operators (replace). Empty for redact.
|
|
388
|
+
*/
|
|
389
|
+
redactionMap: Map<string, string>; /** Maps placeholder to the operator that produced it. */
|
|
390
|
+
operatorMap: Map<string, OperatorType>;
|
|
391
|
+
entityCount: number;
|
|
392
|
+
};
|
|
1066
393
|
//#endregion
|
|
1067
394
|
//#region src/native.d.ts
|
|
1068
395
|
type NativeBindingOperatorConfig = {
|
|
@@ -1123,6 +450,9 @@ type NativeAnonymizeBinding = {
|
|
|
1123
450
|
};
|
|
1124
451
|
prepareStaticSearchPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
1125
452
|
prepareStaticSearchCompressedPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
453
|
+
assembleStaticSearchConfigJson?: (pipelineConfigJson: Uint8Array, dictionariesJson?: Uint8Array, gazetteerJson?: Uint8Array) => Uint8Array;
|
|
454
|
+
assembleStaticSearchPackageBytes?: (pipelineConfigJson: Uint8Array, dictionariesJson?: Uint8Array, gazetteerJson?: Uint8Array) => Uint8Array;
|
|
455
|
+
assembleStaticSearchCompressedPackageBytes?: (pipelineConfigJson: Uint8Array, dictionariesJson?: Uint8Array, gazetteerJson?: Uint8Array) => Uint8Array;
|
|
1126
456
|
};
|
|
1127
457
|
type NativeOperatorConfig = {
|
|
1128
458
|
operators?: Record<string, OperatorType>;
|
|
@@ -1170,6 +500,9 @@ type SharedNativeRedactTextJsonOptions = {
|
|
|
1170
500
|
};
|
|
1171
501
|
type SharedNativeRedactTextOptions = SharedNativeRedactTextJsonOptions;
|
|
1172
502
|
type SharedNativeDiagnosticsJsonOptions = SharedNativeRedactTextJsonOptions;
|
|
503
|
+
type SharedNativeDiagnosticsStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
504
|
+
onBatch: NativeDiagnosticsBatchCallback;
|
|
505
|
+
};
|
|
1173
506
|
type SharedNativeRedactTextStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
1174
507
|
onEvent: NativeResultEventCallback;
|
|
1175
508
|
};
|
|
@@ -1237,11 +570,6 @@ declare class PreparedNativePipeline {
|
|
|
1237
570
|
declare const encodeNativeSearchConfig: (config: NativePreparedSearchConfig) => Uint8Array;
|
|
1238
571
|
declare const encodeNativeSearchConfigInput: (config: NativeSearchPackageInput) => Uint8Array;
|
|
1239
572
|
declare const getNativeBindingVersion: (binding: NativeAnonymizeBinding) => string;
|
|
1240
|
-
declare const native_package_version: (binding: NativeAnonymizeBinding) => string;
|
|
1241
|
-
declare const normalize_for_search: ({
|
|
1242
|
-
binding,
|
|
1243
|
-
text
|
|
1244
|
-
}: NativeNormalizeOptions) => string;
|
|
1245
573
|
declare const assertNativeBindingVersion: ({
|
|
1246
574
|
binding,
|
|
1247
575
|
expectedVersion
|
|
@@ -1251,11 +579,6 @@ declare const prepareNativeSearchPackage: ({
|
|
|
1251
579
|
config,
|
|
1252
580
|
compressed
|
|
1253
581
|
}: NativeSearchPackageOptions) => Uint8Array;
|
|
1254
|
-
declare const prepare_search_package: ({
|
|
1255
|
-
binding,
|
|
1256
|
-
config,
|
|
1257
|
-
compressed
|
|
1258
|
-
}: SharedNativeSearchPackageOptions) => Uint8Array;
|
|
1259
582
|
declare const createNativeAnonymizerFromConfig: ({
|
|
1260
583
|
binding,
|
|
1261
584
|
config
|
|
@@ -1264,117 +587,16 @@ declare const createNativeAnonymizerFromPackage: ({
|
|
|
1264
587
|
binding,
|
|
1265
588
|
packageBytes
|
|
1266
589
|
}: NativeAnonymizerFromPackageOptions) => PreparedNativeAnonymizer;
|
|
1267
|
-
declare const load_prepared_package: ({
|
|
1268
|
-
binding,
|
|
1269
|
-
packageBytes
|
|
1270
|
-
}: SharedNativePreparedPackageOptions) => PreparedNativeAnonymizer;
|
|
1271
|
-
declare const redact_text_json: ({
|
|
1272
|
-
binding,
|
|
1273
|
-
config,
|
|
1274
|
-
fullText,
|
|
1275
|
-
operators
|
|
1276
|
-
}: SharedNativeRedactTextJsonOptions) => string;
|
|
1277
|
-
declare const redact_text: ({
|
|
1278
|
-
binding,
|
|
1279
|
-
config,
|
|
1280
|
-
fullText,
|
|
1281
|
-
operators
|
|
1282
|
-
}: SharedNativeRedactTextOptions) => NativeStaticRedactionResult;
|
|
1283
|
-
declare const redact_text_stream_json: ({
|
|
1284
|
-
binding,
|
|
1285
|
-
config,
|
|
1286
|
-
fullText,
|
|
1287
|
-
operators,
|
|
1288
|
-
onEvent
|
|
1289
|
-
}: SharedNativeRedactTextStreamJsonOptions) => string | null;
|
|
1290
|
-
declare const diagnostics_json: ({
|
|
1291
|
-
binding,
|
|
1292
|
-
config,
|
|
1293
|
-
fullText,
|
|
1294
|
-
operators
|
|
1295
|
-
}: SharedNativeDiagnosticsJsonOptions) => string | null;
|
|
1296
590
|
declare const createNativePipelineFromPackage: ({
|
|
1297
591
|
binding,
|
|
1298
592
|
packageBytes
|
|
1299
593
|
}: NativePipelineFromPackageOptions) => PreparedNativePipeline;
|
|
1300
594
|
declare const PreparedSearch: typeof PreparedNativeAnonymizer;
|
|
1301
595
|
type PreparedSearch = PreparedNativeAnonymizer;
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
declare const DEFAULT_NATIVE_PIPELINE_CONFIG: PipelineConfig;
|
|
1305
|
-
//#endregion
|
|
1306
|
-
//#region src/native-pipeline.d.ts
|
|
1307
|
-
type NativePipelineUnsupportedFeature = "enableNer";
|
|
1308
|
-
type NativePipelineCompatibility = {
|
|
1309
|
-
status: "supported";
|
|
1310
|
-
} | {
|
|
1311
|
-
status: "unsupported";
|
|
1312
|
-
unsupportedFeatures: NativePipelineUnsupportedFeature[];
|
|
1313
|
-
};
|
|
1314
|
-
type NativePipelineBuildOptions = {
|
|
1315
|
-
binding: NativeAnonymizeBinding;
|
|
1316
|
-
config: PipelineConfig;
|
|
1317
|
-
gazetteerEntries?: GazetteerEntry[];
|
|
1318
|
-
context?: PipelineContext;
|
|
1319
|
-
};
|
|
1320
|
-
type NativePipelinePackageOptions = NativePipelineBuildOptions & {
|
|
1321
|
-
compressed?: boolean;
|
|
1322
|
-
};
|
|
1323
|
-
declare const getNativePipelineCompatibility: (config: PipelineConfig) => NativePipelineCompatibility;
|
|
1324
|
-
declare const assertNativePipelineSupported: (config: PipelineConfig) => void;
|
|
1325
|
-
declare const prepareNativePipelineConfig: ({
|
|
1326
|
-
config,
|
|
1327
|
-
gazetteerEntries,
|
|
1328
|
-
context
|
|
1329
|
-
}: Omit<NativePipelineBuildOptions, "binding">) => Promise<NativePreparedSearchConfig>;
|
|
1330
|
-
declare const prepareNativePipelinePackage: ({
|
|
1331
|
-
binding,
|
|
1332
|
-
config,
|
|
1333
|
-
gazetteerEntries,
|
|
1334
|
-
context,
|
|
1335
|
-
compressed
|
|
1336
|
-
}: NativePipelinePackageOptions) => Promise<Uint8Array>;
|
|
1337
|
-
declare const createNativePipelineFromConfig: ({
|
|
1338
|
-
binding,
|
|
1339
|
-
config,
|
|
1340
|
-
gazetteerEntries,
|
|
1341
|
-
context
|
|
1342
|
-
}: NativePipelineBuildOptions) => Promise<PreparedNativePipeline>;
|
|
596
|
+
declare const PreparedAnonymizer: typeof PreparedNativeAnonymizer;
|
|
597
|
+
type PreparedAnonymizer = PreparedNativeAnonymizer;
|
|
1343
598
|
//#endregion
|
|
1344
599
|
//#region src/redact.d.ts
|
|
1345
|
-
/**
|
|
1346
|
-
* Build a stable mapping from entity text to numbered
|
|
1347
|
-
* placeholders. Same real-world value always maps to the
|
|
1348
|
-
* same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
|
|
1349
|
-
* share one person placeholder).
|
|
1350
|
-
*
|
|
1351
|
-
* Placeholder format: [LABEL_N] where LABEL is uppercase.
|
|
1352
|
-
* N is allocated per label and skips tokens already present
|
|
1353
|
-
* in reserved text.
|
|
1354
|
-
*
|
|
1355
|
-
* @param _ctx Unused. Kept for signature compatibility;
|
|
1356
|
-
* coref alias links now travel on the entities
|
|
1357
|
-
* themselves (`corefSourceText`).
|
|
1358
|
-
*/
|
|
1359
|
-
type PlaceholderMapOptions = {
|
|
1360
|
-
reservedText?: string;
|
|
1361
|
-
};
|
|
1362
|
-
declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext, {
|
|
1363
|
-
reservedText
|
|
1364
|
-
}?: PlaceholderMapOptions) => Map<string, string>;
|
|
1365
|
-
/**
|
|
1366
|
-
* Apply redactions to the source text, replacing each
|
|
1367
|
-
* confirmed entity span using the configured operator.
|
|
1368
|
-
*
|
|
1369
|
-
* Co-references are consistent: if the same text appears
|
|
1370
|
-
* multiple times, all occurrences get the same placeholder.
|
|
1371
|
-
*
|
|
1372
|
-
* @param ctx Pipeline context. Must be the same instance
|
|
1373
|
-
* passed to `runPipeline` (or `findCoreferenceSpans`)
|
|
1374
|
-
* so coreference placeholder links are preserved.
|
|
1375
|
-
* Defaults to `defaultContext` for single-tenant usage.
|
|
1376
|
-
*/
|
|
1377
|
-
declare const redactText: (fullText: string, entities: Entity[], config?: OperatorConfig, ctx?: PipelineContext) => RedactionResult;
|
|
1378
600
|
/**
|
|
1379
601
|
* Serialize the redaction key to JSON for export.
|
|
1380
602
|
* Includes operator metadata so the export is self-describing.
|
|
@@ -1387,457 +609,60 @@ declare const exportRedactionKey: (redactionMap: Map<string, string>, operatorMa
|
|
|
1387
609
|
*/
|
|
1388
610
|
declare const deanonymise: (redactedText: string, redactionMap: Map<string, string>) => string;
|
|
1389
611
|
//#endregion
|
|
1390
|
-
//#region src/
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
*
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
/**
|
|
1401
|
-
*
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
*
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
*
|
|
1416
|
-
*
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
*
|
|
1420
|
-
*
|
|
1421
|
-
*
|
|
1422
|
-
* `
|
|
1423
|
-
*
|
|
1424
|
-
*
|
|
1425
|
-
*
|
|
1426
|
-
|
|
1427
|
-
declare const
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
patterns: string[];
|
|
1434
|
-
rules: TriggerRule[];
|
|
1435
|
-
};
|
|
1436
|
-
declare const buildTriggerPatterns: (languages?: readonly string[]) => Promise<TriggerPatterns>;
|
|
1437
|
-
/**
|
|
1438
|
-
* Process trigger matches from the unified search.
|
|
1439
|
-
* Receives all matches; filters to the trigger slice
|
|
1440
|
-
* via sliceStart/sliceEnd. Uses fullText for value
|
|
1441
|
-
* extraction (the unified search runs on lowercased
|
|
1442
|
-
* text, but extraction needs original casing).
|
|
1443
|
-
*/
|
|
1444
|
-
declare const processTriggerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, rules: readonly TriggerRule[]) => Entity[];
|
|
1445
|
-
//#endregion
|
|
1446
|
-
//#region src/detectors/gazetteer.d.ts
|
|
1447
|
-
/**
|
|
1448
|
-
* Build TextSearch-compatible patterns from gazetteer
|
|
1449
|
-
* entries. Returns:
|
|
1450
|
-
* - Exact literal patterns for all terms
|
|
1451
|
-
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
1452
|
-
* - Parallel metadata arrays for post-processing
|
|
1453
|
-
*
|
|
1454
|
-
* Patterns are ordered: all exact first, then all
|
|
1455
|
-
* fuzzy. The isFuzzy array marks which are which.
|
|
1456
|
-
*/
|
|
1457
|
-
declare const buildGazetteerPatterns: (entries: GazetteerEntry[]) => {
|
|
1458
|
-
patterns: PatternEntry[];
|
|
1459
|
-
data: GazetteerData;
|
|
1460
|
-
};
|
|
1461
|
-
/**
|
|
1462
|
-
* Process gazetteer matches from the unified literal
|
|
1463
|
-
* search. Receives all matches; filters to the
|
|
1464
|
-
* gazetteer slice via sliceStart/sliceEnd.
|
|
1465
|
-
*
|
|
1466
|
-
* Exact matches get score 0.9; fuzzy matches get
|
|
1467
|
-
* 0.85. Fuzzy matches that overlap an exact match
|
|
1468
|
-
* are dropped.
|
|
1469
|
-
*
|
|
1470
|
-
* For exact matches, attempts prefix extension for
|
|
1471
|
-
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
1472
|
-
* the matched term).
|
|
1473
|
-
*/
|
|
1474
|
-
declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: GazetteerData) => Entity[];
|
|
1475
|
-
//#endregion
|
|
1476
|
-
//#region src/detectors/coreference.d.ts
|
|
1477
|
-
type ExtractDefinedTermsOptions = {
|
|
1478
|
-
languages?: readonly string[];
|
|
1479
|
-
};
|
|
1480
|
-
type DefinedTerm = {
|
|
1481
|
-
alias: string;
|
|
1482
|
-
label: string; /** Position of the definition in the source text */
|
|
1483
|
-
definitionStart: number; /** Original entity text the alias refers to */
|
|
1484
|
-
sourceText: string;
|
|
1485
|
-
};
|
|
1486
|
-
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext, options?: ExtractDefinedTermsOptions) => Promise<DefinedTerm[]>;
|
|
1487
|
-
/**
|
|
1488
|
-
* Find all occurrences of defined-term aliases in the
|
|
1489
|
-
* full text. Returns Entity spans for each match.
|
|
1490
|
-
*
|
|
1491
|
-
* Respects word boundaries: "Kupující" must not match
|
|
1492
|
-
* inside "Kupujícímu". A match is valid only if the
|
|
1493
|
-
* character before the start and after the end are NOT
|
|
1494
|
-
* word characters (letter/digit).
|
|
1495
|
-
*
|
|
1496
|
-
* Each returned alias carries `corefSourceText` linking
|
|
1497
|
-
* it to its source entity text, for consistent
|
|
1498
|
-
* placeholder numbering.
|
|
1499
|
-
*
|
|
1500
|
-
* @param _ctx Unused. Kept for signature compatibility;
|
|
1501
|
-
* alias links now travel on the entities themselves.
|
|
1502
|
-
*/
|
|
1503
|
-
declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ctx?: PipelineContext) => Entity[];
|
|
1504
|
-
//#endregion
|
|
1505
|
-
//#region src/detectors/org-propagation.d.ts
|
|
1506
|
-
/**
|
|
1507
|
-
* After the main detection pass, collect organization
|
|
1508
|
-
* entities with a legal form suffix, strip the suffix
|
|
1509
|
-
* to get the base name, and re-scan the full text for
|
|
1510
|
-
* bare mentions of that base name. Returns new entities
|
|
1511
|
-
* for occurrences not already covered.
|
|
1512
|
-
*
|
|
1513
|
-
* Propagated mentions are coref aliases: each carries
|
|
1514
|
-
* `corefSourceText` linking it to the full seed entity
|
|
1515
|
-
* text, so placeholder numbering assigns the bare
|
|
1516
|
-
* mention the same placeholder as its source ("Acme"
|
|
1517
|
-
* and "Acme Corp." both become [ORGANIZATION_1]).
|
|
1518
|
-
*/
|
|
1519
|
-
declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
|
|
1520
|
-
//#endregion
|
|
1521
|
-
//#region src/detectors/names.d.ts
|
|
1522
|
-
declare const getNameCorpusNonWesternNames: (ctx?: PipelineContext) => readonly string[];
|
|
1523
|
-
/**
|
|
1524
|
-
* Load name corpus data from injected dictionaries
|
|
1525
|
-
* and legacy config files. Merges all sources.
|
|
1526
|
-
*
|
|
1527
|
-
* Safe to call multiple times; only loads once per
|
|
1528
|
-
* context. Must be called before detectNameCorpus or
|
|
1529
|
-
* the getNameCorpus*() accessors are used.
|
|
1530
|
-
*
|
|
1531
|
-
* @param dictionaries Optional pre-loaded dictionaries
|
|
1532
|
-
* with per-language first names and surnames. When
|
|
1533
|
-
* omitted, only legacy config files are used.
|
|
1534
|
-
*/
|
|
1535
|
-
declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionaries, languages?: readonly string[]) => Promise<void>;
|
|
1536
|
-
type NameCorpusDetectionOptions = {
|
|
1537
|
-
mode?: "full" | "supplemental";
|
|
1538
|
-
};
|
|
1539
|
-
/**
|
|
1540
|
-
* Detect person names by looking up tokens against the
|
|
1541
|
-
* name corpus, then chaining adjacent name-like tokens.
|
|
1542
|
-
* Handles both Western and non-Western name patterns.
|
|
1543
|
-
*
|
|
1544
|
-
* Requires initNameCorpus() to have been called first.
|
|
1545
|
-
* If not initialized, returns an empty array.
|
|
1546
|
-
*
|
|
1547
|
-
* Scoring (Western):
|
|
1548
|
-
* TITLE + NAME/SURNAME → 0.95
|
|
1549
|
-
* NAME + NAME/SURNAME → 0.9
|
|
1550
|
-
* SURNAME + NAME/SURNAME → 0.9
|
|
1551
|
-
* NAME + CAPITALIZED → 0.7
|
|
1552
|
-
* ABBREVIATION + NAME → 0.7
|
|
1553
|
-
* Standalone NAME → 0.5 (low confidence)
|
|
1554
|
-
* Standalone SURNAME → skip (too ambiguous)
|
|
1555
|
-
*
|
|
1556
|
-
* Scoring (non-Western, when chain contains nonWestern tokens):
|
|
1557
|
-
* TITLE + (nonWestern|CAPITALIZED) → 0.95
|
|
1558
|
-
* JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
|
|
1559
|
-
* ARABIC_CONNECTOR + nonWestern → 0.9
|
|
1560
|
-
* 2+ nonWestern tokens → 0.9
|
|
1561
|
-
* nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
|
|
1562
|
-
* Standalone nonWestern mid-sentence → 0.5
|
|
1563
|
-
*/
|
|
1564
|
-
declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext, options?: NameCorpusDetectionOptions) => Entity[];
|
|
1565
|
-
//#endregion
|
|
1566
|
-
//#region src/unified-search.d.ts
|
|
1567
|
-
type UnifiedResult = {
|
|
1568
|
-
/** All matches from both instances combined. */regexMatches: Match[];
|
|
1569
|
-
customRegexMatches: Match[];
|
|
1570
|
-
literalMatches: Match[];
|
|
1571
|
-
};
|
|
1572
|
-
declare const runUnifiedSearch: (instance: UnifiedSearchInstance, fullText: string) => UnifiedResult;
|
|
1573
|
-
//#endregion
|
|
1574
|
-
//#region src/regions.d.ts
|
|
1575
|
-
/**
|
|
1576
|
-
* Geographic regions and country code mappings for
|
|
1577
|
-
* scoping deny list dictionaries.
|
|
1578
|
-
*/
|
|
1579
|
-
declare const REGIONS: {
|
|
1580
|
-
readonly Global: null;
|
|
1581
|
-
readonly International: null;
|
|
1582
|
-
readonly Europe: readonly ["AL", "AD", "AT", "BE", "BA", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IS", "IE", "IT", "XK", "LV", "LI", "LT", "LU", "MD", "ME", "MK", "MT", "MC", "NL", "NO", "PL", "PT", "RO", "RS", "SK", "SI", "ES", "SE", "CH", "UA", "GB"];
|
|
1583
|
-
readonly Americas: readonly ["US", "CA", "MX", "BR", "AR", "CL", "CO", "PE", "EC", "VE", "UY", "PY", "BO", "CR", "PA", "DO", "GT", "HN", "SV", "NI", "CU"];
|
|
1584
|
-
readonly AsiaPacific: readonly ["AU", "NZ", "JP", "KR", "CN", "TW", "SG", "MY", "TH", "VN", "PH", "ID", "IN", "PK", "BD", "LK", "NP", "HK", "MO"];
|
|
1585
|
-
readonly MENA: readonly ["AE", "SA", "IL", "TR", "EG", "JO", "LB", "IQ", "IR", "QA", "KW", "BH", "OM", "MA", "TN", "DZ", "LY", "SY", "YE", "PS"];
|
|
1586
|
-
readonly SubSaharanAfrica: readonly ["ZA", "NG", "KE", "GH", "TZ", "ET", "SN", "CI", "CM", "UG", "RW", "MZ", "AO", "ZW", "BW", "NA", "MU"];
|
|
1587
|
-
readonly EU: readonly ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE"];
|
|
1588
|
-
readonly DACH: readonly ["DE", "AT", "CH"];
|
|
1589
|
-
readonly Nordics: readonly ["DK", "SE", "NO", "FI", "IS"];
|
|
1590
|
-
readonly CEE: readonly ["CZ", "SK", "PL", "HU", "RO", "BG", "HR", "SI", "LT", "LV", "EE"];
|
|
1591
|
-
readonly Anglosphere: readonly ["GB", "US", "CA", "AU", "NZ", "IE"];
|
|
1592
|
-
readonly Benelux: readonly ["BE", "NL", "LU"];
|
|
1593
|
-
readonly GulfStates: readonly ["AE", "SA", "QA", "KW", "BH", "OM"];
|
|
1594
|
-
readonly SouthAsia: readonly ["IN", "PK", "BD", "LK", "NP"];
|
|
1595
|
-
readonly EastAsia: readonly ["CN", "JP", "KR", "TW"];
|
|
1596
|
-
readonly SoutheastAsia: readonly ["SG", "MY", "TH", "VN", "PH", "ID"];
|
|
1597
|
-
readonly Oceania: readonly ["AU", "NZ"];
|
|
1598
|
-
};
|
|
1599
|
-
type RegionId = keyof typeof REGIONS;
|
|
1600
|
-
type RegionArrays = { [K in RegionId]: (typeof REGIONS)[K] };
|
|
1601
|
-
type NonNullRegion = { [K in RegionId as RegionArrays[K] extends null ? never : K]: RegionArrays[K] };
|
|
1602
|
-
type CountryCode = NonNullRegion[keyof NonNullRegion][number];
|
|
1603
|
-
/**
|
|
1604
|
-
* Expand region names to country codes and merge with
|
|
1605
|
-
* explicit country codes. Returns null when both inputs
|
|
1606
|
-
* are empty/undefined (meaning "match all countries").
|
|
1607
|
-
*/
|
|
1608
|
-
declare const resolveCountries: (regions?: string[], countries?: string[]) => Set<string> | null;
|
|
1609
|
-
//#endregion
|
|
1610
|
-
//#region src/filters/false-positives.d.ts
|
|
1611
|
-
/** Ensure street-type vocabulary is loaded. */
|
|
1612
|
-
declare const initAddressComponents: () => Promise<void>;
|
|
1613
|
-
/**
|
|
1614
|
-
* Filter out entities that are likely false positives:
|
|
1615
|
-
* template placeholders, clause/section numbers,
|
|
1616
|
-
* standalone years, and generic legal role terms.
|
|
1617
|
-
*
|
|
1618
|
-
* Runs as a post-processing step after all detection
|
|
1619
|
-
* layers have merged.
|
|
1620
|
-
*/
|
|
1621
|
-
declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext, fullText?: string) => Entity[];
|
|
1622
|
-
//#endregion
|
|
1623
|
-
//#region src/filters/hotword-rules.d.ts
|
|
1624
|
-
type HotwordRule = {
|
|
1625
|
-
hotwords: string[];
|
|
1626
|
-
targetLabels: string[];
|
|
1627
|
-
scoreAdjustment: number;
|
|
1628
|
-
reclassifyTo?: string;
|
|
1629
|
-
proximityBefore: number;
|
|
1630
|
-
proximityAfter: number;
|
|
1631
|
-
};
|
|
1632
|
-
/**
|
|
1633
|
-
* Load hotword rules from the data package.
|
|
1634
|
-
* Safe to call multiple times; subsequent calls
|
|
1635
|
-
* are no-ops.
|
|
1636
|
-
*/
|
|
1637
|
-
declare const initHotwordRules: () => Promise<void>;
|
|
1638
|
-
/**
|
|
1639
|
-
* Apply hotword context rules to detected entities.
|
|
1640
|
-
*
|
|
1641
|
-
* Scans `fullText` once with a single AC automaton
|
|
1642
|
-
* for all hotwords across all rules, then checks
|
|
1643
|
-
* proximity to each entity. Distance-decayed
|
|
1644
|
-
* adjustment: closer hotwords give a stronger boost.
|
|
1645
|
-
*
|
|
1646
|
-
* Returns a new array; input entities are not mutated.
|
|
1647
|
-
*/
|
|
1648
|
-
declare const applyHotwordRules: (entities: Entity[], fullText: string) => Entity[];
|
|
1649
|
-
//#endregion
|
|
1650
|
-
//#region src/filters/zone-classifier.d.ts
|
|
1651
|
-
type DocumentZone = "header" | "signature" | "body" | "table";
|
|
1652
|
-
type ZoneSpan = {
|
|
1653
|
-
zone: DocumentZone;
|
|
1654
|
-
start: number;
|
|
1655
|
-
end: number;
|
|
1656
|
-
};
|
|
1657
|
-
/**
|
|
1658
|
-
* Additive score adjustments per document zone.
|
|
1659
|
-
* Header and signature blocks are dense with PII;
|
|
1660
|
-
* tables often contain structured identifying data.
|
|
1661
|
-
*/
|
|
1662
|
-
declare const ZONE_SCORE_ADJUSTMENTS: {
|
|
1663
|
-
readonly header: 0.1;
|
|
1664
|
-
readonly signature: 0.15;
|
|
1665
|
-
readonly body: 0;
|
|
1666
|
-
readonly table: 0.05;
|
|
1667
|
-
};
|
|
1668
|
-
/**
|
|
1669
|
-
* Ensure config data is loaded. Call once before
|
|
1670
|
-
* classifyZones. Safe to call multiple times.
|
|
1671
|
-
*/
|
|
1672
|
-
declare const initZoneClassifier: (ctx?: PipelineContext) => Promise<void>;
|
|
1673
|
-
/**
|
|
1674
|
-
* Classify a document into zones based on
|
|
1675
|
-
* structural heuristics. Zones are non-overlapping
|
|
1676
|
-
* and cover the entire text.
|
|
1677
|
-
*
|
|
1678
|
-
* Must call `initZoneClassifier()` first.
|
|
1679
|
-
*/
|
|
1680
|
-
declare const classifyZones: (fullText: string, ctx?: PipelineContext) => ZoneSpan[];
|
|
1681
|
-
/**
|
|
1682
|
-
* Apply zone-based score adjustments to entities.
|
|
1683
|
-
* Entities in header/signature/table zones get a
|
|
1684
|
-
* small additive boost reflecting the higher PII
|
|
1685
|
-
* density in those regions.
|
|
1686
|
-
*
|
|
1687
|
-
* Returns a new array; does not mutate inputs.
|
|
1688
|
-
*/
|
|
1689
|
-
declare const applyZoneAdjustments: (entities: Entity[], zones: ZoneSpan[]) => Entity[];
|
|
1690
|
-
//#endregion
|
|
1691
|
-
//#region src/gliner/types.d.ts
|
|
1692
|
-
/**
|
|
1693
|
-
* GLiNER inference types.
|
|
1694
|
-
*
|
|
1695
|
-
* Forked from gliner@0.0.19 (MIT), stripped to runtime-
|
|
1696
|
-
* agnostic core. Original: github.com/Ingvarstep/GLiNER.js
|
|
1697
|
-
*/
|
|
1698
|
-
type EntityResult = {
|
|
1699
|
-
spanText: string;
|
|
1700
|
-
start: number;
|
|
1701
|
-
end: number;
|
|
1702
|
-
label: string;
|
|
1703
|
-
score: number;
|
|
1704
|
-
};
|
|
1705
|
-
/**
|
|
1706
|
-
* Raw inference output: per-batch array of
|
|
1707
|
-
* [spanText, start, end, label, score] tuples.
|
|
1708
|
-
*/
|
|
1709
|
-
type RawInferenceResult = [string, number, number, string, number][][];
|
|
1710
|
-
//#endregion
|
|
1711
|
-
//#region src/gliner/decoder.d.ts
|
|
1712
|
-
/**
|
|
1713
|
-
* Decode span-level model logits into entity results.
|
|
1714
|
-
*/
|
|
1715
|
-
declare const decodeSpans: (batchSize: number, inputLength: number, maxWidth: number, numEntities: number, texts: string[], batchIds: number[], batchWordsStartIdx: number[][], batchWordsEndIdx: number[][], idToClass: Record<number, string>, modelOutput: ArrayLike<number>, flatNer: boolean, threshold: number, multiLabel: boolean) => RawInferenceResult;
|
|
1716
|
-
//#endregion
|
|
1717
|
-
//#region src/gliner/token-decoder.d.ts
|
|
1718
|
-
/**
|
|
1719
|
-
* Decode token-level BIO logits into entity spans.
|
|
1720
|
-
*
|
|
1721
|
-
* For each word, checks if the B(egin) logit for any class
|
|
1722
|
-
* exceeds the threshold. If so, extends the span by consuming
|
|
1723
|
-
* subsequent I(nside) tokens of the same class.
|
|
1724
|
-
*/
|
|
1725
|
-
declare const decodeTokenSpans: (batchSize: number, numWords: number, numEntities: number, texts: string[], batchIds: number[], batchWordsStartIdx: number[][], batchWordsEndIdx: number[][], idToClass: Record<number, string>, modelOutput: ArrayLike<number>, threshold: number) => RawInferenceResult;
|
|
1726
|
-
//#endregion
|
|
1727
|
-
//#region src/gliner/processor.d.ts
|
|
1728
|
-
/** Tokenize text into words with character offsets. */
|
|
1729
|
-
declare const tokenizeText: (text: string) => [words: string[], starts: number[], ends: number[]];
|
|
1730
|
-
/** Prepare a complete batch for ONNX inference. */
|
|
1731
|
-
declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: string[], maxWidth: number) => {
|
|
1732
|
-
inputsIds: number[][];
|
|
1733
|
-
attentionMasks: number[][];
|
|
1734
|
-
wordsMasks: number[][];
|
|
1735
|
-
textLengths: number[];
|
|
1736
|
-
spanIdxs: number[][][];
|
|
1737
|
-
spanMasks: boolean[][];
|
|
1738
|
-
idToClass: Record<number, string>;
|
|
1739
|
-
batchTokens: string[][];
|
|
1740
|
-
batchWordsStartIdx: number[][];
|
|
1741
|
-
batchWordsEndIdx: number[][];
|
|
1742
|
-
};
|
|
1743
|
-
//#endregion
|
|
1744
|
-
//#region src/util/chunker.d.ts
|
|
1745
|
-
/** A chunk paired with its start offset in the source text. */
|
|
1746
|
-
type ChunkSpan = {
|
|
1747
|
-
text: string;
|
|
1748
|
-
offset: number;
|
|
612
|
+
//#region src/wasm.d.ts
|
|
613
|
+
/** A prepared package the caller supplies: raw bytes, an ArrayBuffer, or a URL
|
|
614
|
+
* (string or `URL`) that resolves to the package and is fetched. */
|
|
615
|
+
type PreparedPackageSource = Uint8Array | ArrayBuffer | URL | string;
|
|
616
|
+
/** Escape hatch for callers that already hold a binding (e.g. a custom sidecar
|
|
617
|
+
* or a test double). When omitted, the lazily-instantiated wasm binding is
|
|
618
|
+
* used. */
|
|
619
|
+
type WasmBindingOptions = {
|
|
620
|
+
binding?: NativeAnonymizeBinding;
|
|
621
|
+
};
|
|
622
|
+
/** Instantiate (once) and return the wasm binding. Safe to call repeatedly:
|
|
623
|
+
* the underlying wasm module is instantiated a single time and cached. */
|
|
624
|
+
declare const getBinding: () => Promise<NativeAnonymizeBinding>;
|
|
625
|
+
type LoadPreparedPackageOptions = WasmBindingOptions;
|
|
626
|
+
/** Load a prepared package and return a pipeline ready to redact text. */
|
|
627
|
+
declare const loadPipeline: (source: PreparedPackageSource, options?: LoadPreparedPackageOptions) => Promise<PreparedNativePipeline>;
|
|
628
|
+
/** Load a prepared package and return the lower-level anonymizer. */
|
|
629
|
+
declare const load_prepared_package: (source: PreparedPackageSource, options?: LoadPreparedPackageOptions) => Promise<PreparedNativeAnonymizer>;
|
|
630
|
+
/** URL of a bundled default prepared package, resolved against this module so
|
|
631
|
+
* it points at the `native/` asset directory shipped in the tarball. */
|
|
632
|
+
declare const defaultPackageUrl: (language?: string) => URL;
|
|
633
|
+
/** Load a fresh pipeline from the bundled default prepared package.
|
|
634
|
+
*
|
|
635
|
+
* Mirrors the node loader's regional-tag fallback: when an exact package for
|
|
636
|
+
* a locale tag such as `en-US` is not shipped, the base-language package
|
|
637
|
+
* (`en`) is loaded instead. The browser cannot check asset existence up
|
|
638
|
+
* front, so the fallback triggers on a failed load of the exact package. */
|
|
639
|
+
declare const loadDefaultPipeline: (language?: string, options?: LoadPreparedPackageOptions) => Promise<PreparedNativePipeline>;
|
|
640
|
+
/** Cached variant of {@link loadDefaultPipeline}: the default pipeline for a
|
|
641
|
+
* given language is fetched and prepared once, then reused.
|
|
642
|
+
*
|
|
643
|
+
* Only the ambient-binding case is cached. The cache key is language-only, so a
|
|
644
|
+
* caller that injects its own `options.binding` bypasses the cache entirely:
|
|
645
|
+
* reusing a pipeline built against a different binding would be wrong, and
|
|
646
|
+
* folding the binding into the key would keep unbounded per-binding entries
|
|
647
|
+
* alive. Injected-binding callers get a fresh pipeline each call. */
|
|
648
|
+
declare const getDefaultPipeline: (language?: string, options?: LoadPreparedPackageOptions) => Promise<PreparedNativePipeline>;
|
|
649
|
+
declare const redactDefaultText: (fullText: string, operators?: NativeOperatorConfig, language?: string) => Promise<NativeStaticRedactionResult>;
|
|
650
|
+
declare const redactDefaultTextJson: (fullText: string, operators?: NativeOperatorConfig, language?: string) => Promise<string>;
|
|
651
|
+
declare const native_package_version: (options?: WasmBindingOptions) => Promise<string>;
|
|
652
|
+
declare const normalize_for_search: (text: string, options?: WasmBindingOptions) => Promise<string>;
|
|
653
|
+
type PrepareSearchPackageOptions = WasmBindingOptions & {
|
|
654
|
+
compressed?: boolean;
|
|
1749
655
|
};
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
* when possible.
|
|
1763
|
-
*/
|
|
1764
|
-
declare const chunkTextWithOffsets: (text: string) => ChunkSpan[];
|
|
1765
|
-
/**
|
|
1766
|
-
* Split text into overlapping chunks for GLiNER's ~512 token
|
|
1767
|
-
* context window. Character-based splitting (rough token
|
|
1768
|
-
* approximation); breaks at sentence boundaries when possible.
|
|
1769
|
-
*
|
|
1770
|
-
* Prefer chunkTextWithOffsets when you also need each chunk's
|
|
1771
|
-
* document offset.
|
|
1772
|
-
*/
|
|
1773
|
-
declare const chunkText: (text: string) => string[];
|
|
1774
|
-
/**
|
|
1775
|
-
* Compute the start offset of each chunk within the original
|
|
1776
|
-
* document text by content search.
|
|
1777
|
-
*
|
|
1778
|
-
* @deprecated Re-locates each chunk with `indexOf`, which can
|
|
1779
|
-
* match the wrong position when identical content repeats in
|
|
1780
|
-
* the document (common in boilerplate-heavy legal text) and
|
|
1781
|
-
* then desyncs every subsequent offset. Use
|
|
1782
|
-
* `chunkTextWithOffsets`, which carries exact offsets out of
|
|
1783
|
-
* the splitter.
|
|
1784
|
-
*/
|
|
1785
|
-
declare const computeChunkOffsets: (fullText: string, chunks: string[]) => number[];
|
|
1786
|
-
/**
|
|
1787
|
-
* Merge entities from overlapping chunks back to
|
|
1788
|
-
* document-level offsets. Deduplicates entities that
|
|
1789
|
-
* appear in overlap regions (keeps highest score).
|
|
1790
|
-
*
|
|
1791
|
-
* Dedup invariant: each incoming entity is compared
|
|
1792
|
-
* against the highest-scored same-label near-dup in
|
|
1793
|
-
* its proximity window. If it loses, it is dropped.
|
|
1794
|
-
* This does NOT guarantee that all pairwise near-dup
|
|
1795
|
-
* relationships in the output are resolved; a lower-
|
|
1796
|
-
* scored entity can survive if the bridging entity
|
|
1797
|
-
* that would have replaced it was itself dropped by
|
|
1798
|
-
* a higher-scored match.
|
|
1799
|
-
*
|
|
1800
|
-
* Uses a reverse-scan over the sorted merged array
|
|
1801
|
-
* so each entity only compares against nearby
|
|
1802
|
-
* predecessors — O(n * w) average where w is the max
|
|
1803
|
-
* entities per POSITION_THRESHOLD window, O(n²) worst
|
|
1804
|
-
* case when replacements dominate (splice is O(n)).
|
|
1805
|
-
*/
|
|
1806
|
-
declare const mergeChunkEntities: (chunkOffsets: number[], chunkResults: Entity[][]) => Entity[];
|
|
1807
|
-
//#endregion
|
|
1808
|
-
//#region src/util/levenshtein.d.ts
|
|
1809
|
-
/**
|
|
1810
|
-
* Compute the Levenshtein edit distance between two
|
|
1811
|
-
* strings. O(n*m) time, O(min(n,m)) space using a
|
|
1812
|
-
* single-row DP approach.
|
|
1813
|
-
*/
|
|
1814
|
-
declare const levenshtein: (rawA: string, rawB: string) => number;
|
|
1815
|
-
//#endregion
|
|
1816
|
-
//#region src/util/normalize.d.ts
|
|
1817
|
-
/**
|
|
1818
|
-
* Normalize typographic variants for search matching.
|
|
1819
|
-
*
|
|
1820
|
-
* Legal documents (especially Czech/German) use
|
|
1821
|
-
* non-breaking spaces, smart quotes, and en/em dashes
|
|
1822
|
-
* that differ from their ASCII equivalents. Since all
|
|
1823
|
-
* replacements are same-length (single code unit →
|
|
1824
|
-
* single code unit), character offsets remain valid.
|
|
1825
|
-
*
|
|
1826
|
-
* Lives here (application layer) rather than in the
|
|
1827
|
-
* AC library: what to normalize is domain-specific.
|
|
1828
|
-
*
|
|
1829
|
-
* Uses a char-code lookup (`Map<number, number>`) and
|
|
1830
|
-
* `Uint16Array` instead of 7 sequential `replaceAll`
|
|
1831
|
-
* calls. For a 50 KB document this eliminates ~350 KB
|
|
1832
|
-
* of intermediate string allocations.
|
|
1833
|
-
*
|
|
1834
|
-
* When no replaceable characters are present (common
|
|
1835
|
-
* for plain-text inputs), a fast-path scan returns the
|
|
1836
|
-
* original string without any allocation. When special
|
|
1837
|
-
* characters exist, the string is scanned twice: once
|
|
1838
|
-
* to detect, once to build the replacement array.
|
|
1839
|
-
*/
|
|
1840
|
-
declare const normalizeForSearch: (text: string) => string;
|
|
1841
|
-
//#endregion
|
|
1842
|
-
export { type AnonymisationOperator, CURRENCY_PATTERN_META, type ChunkSpan, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NativeAnonymizeBinding, type NativeAnonymizerFromConfigOptions, type NativeAnonymizerFromPackageOptions, type NativeBindingVersionOptions, type NativeNormalizeOptions, type NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, type NativePipelineEntity, type NativePipelineFromPackageOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, type NativePreparedSearchBinding, type NativeRedactionResult, type NativeResultEventCallback, type NativeSearchPackageInput, type NativeSearchPackageOptions, type NativeStaticRedactionResult, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, type PreparedSearch as PreparedSearchInstance, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type SharedNativeDiagnosticsJsonOptions, type SharedNativePreparedPackageOptions, type SharedNativeRedactTextJsonOptions, type SharedNativeRedactTextOptions, type SharedNativeRedactTextStreamJsonOptions, type SharedNativeSearchPackageOptions, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, assertNativeBindingVersion, assertNativePipelineSupported, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, chunkTextWithOffsets, classifyZones, computeChunkOffsets, corefKey, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromPackage, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, diagnostics_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, getNativeBindingVersion, getNativePipelineCompatibility, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, load_prepared_package, mergeAndDedup, mergeChunkEntities, native_package_version, normalizeForSearch, normalize_for_search, prepareBatch, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, preparePipelineSearch, prepare_search_package, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, redact_text, redact_text_json, redact_text_stream_json, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
656
|
+
declare const prepare_search_package: (config: NativeSearchPackageInput, {
|
|
657
|
+
compressed,
|
|
658
|
+
...options
|
|
659
|
+
}?: PrepareSearchPackageOptions) => Promise<Uint8Array>;
|
|
660
|
+
declare const redact_text: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<NativeStaticRedactionResult>;
|
|
661
|
+
declare const redact_text_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string>;
|
|
662
|
+
declare const redact_text_stream_json: (config: NativeSearchPackageInput, fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
663
|
+
declare const diagnostics_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
664
|
+
declare const diagnostics_stream_json: (config: NativeSearchPackageInput, fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
665
|
+
declare const summary_diagnostics_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
666
|
+
//#endregion
|
|
667
|
+
export { type AnonymisationOperator, DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, type DetectionSource, type Entity, LoadPreparedPackageOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeDiagnosticsBatchCallback, NativeNormalizeOptions, NativeOperatorConfig, NativePipelineEntity, NativePipelineFromPackageOptions, NativePreparedSearchBinding, type NativePreparedSearchConfig, NativeRedactionResult, NativeResultEventCallback, NativeSearchPackageInput, NativeSearchPackageOptions, NativeStaticRedactionResult, OPERATOR_TYPES, type OperatorConfig, type OperatorType, PrepareSearchPackageOptions, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedPackageSource, PreparedSearch, type RedactionResult, type ReviewDecision, type ReviewedEntity, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, WasmBindingOptions, assertNativeBindingVersion, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromPackage, deanonymise, defaultPackageUrl, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getBinding, getDefaultPipeline, getNativeBindingVersion, loadDefaultPipeline, loadPipeline, load_prepared_package, native_package_version, normalize_for_search, prepareNativeSearchPackage, prepare_search_package, redactDefaultText, redactDefaultTextJson, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
1843
668
|
//# sourceMappingURL=wasm.d.mts.map
|