@stll/anonymize 1.5.0 → 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +70 -0
- package/README.md +86 -48
- package/dist/address-boundaries.mjs +2 -0
- package/dist/address-jurisdiction-prefixes.mjs +16 -0
- package/dist/address-jurisdiction-prefixes.mjs.map +1 -0
- package/dist/address-stop-keywords.mjs +12 -1
- package/dist/address-unit-abbreviations.mjs +15 -0
- package/dist/address-unit-abbreviations.mjs.map +1 -0
- package/dist/clause-noun-heads.mjs +5 -1
- package/dist/coreference-org-determiners.mjs +19 -0
- package/dist/coreference-org-determiners.mjs.map +1 -0
- package/dist/defined-term-heads.mjs +15 -0
- package/dist/defined-term-heads.mjs.map +1 -0
- package/dist/false-positive-shapes.mjs +36 -0
- package/dist/false-positive-shapes.mjs.map +1 -0
- package/dist/index.d.mts +3 -1201
- package/dist/index.mjs +3 -16267
- package/dist/index.mjs.map +1 -1
- package/dist/legal-role-heads.cs.mjs +6 -0
- package/dist/native-node.d.mts +124 -0
- package/dist/native-node.mjs +3 -0
- package/dist/native-node2.d.mts +3 -0
- package/dist/native-node2.mjs +13353 -0
- package/dist/native-node2.mjs.map +1 -0
- package/dist/native.d.mts +1158 -0
- package/dist/native.mjs +230 -0
- package/dist/native.mjs.map +1 -0
- package/dist/native2.d.mts +2 -0
- package/dist/organization-unit-heads.mjs +20 -0
- package/dist/organization-unit-heads.mjs.map +1 -0
- package/dist/person-stopwords.mjs +205 -199
- package/dist/signing-clauses.mjs +33 -9
- package/index.cjs +3 -0
- package/native-pipeline.cs.stlanonpkg +0 -0
- package/native-pipeline.de.stlanonpkg +0 -0
- package/native-pipeline.en.stlanonpkg +0 -0
- package/native-pipeline.stlanonpkg +0 -0
- package/package.json +40 -6
- package/scripts/build-native-pipeline-package.mjs +225 -0
- package/dist/address-prepositions.mjs +0 -182
- package/dist/address-prepositions.mjs.map +0 -1
|
@@ -0,0 +1,1158 @@
|
|
|
1
|
+
import { i as DetectionSource, n as DETECTION_SOURCES, o as OperatorType } from "./constants2.mjs";
|
|
2
|
+
import { Validator } from "@stll/stdnum";
|
|
3
|
+
import { TextSearch } from "@stll/text-search";
|
|
4
|
+
|
|
5
|
+
//#region src/types.d.ts
|
|
6
|
+
/**
|
|
7
|
+
* Fields shared by every entity span in the source text.
|
|
8
|
+
*/
|
|
9
|
+
type EntityBase = {
|
|
10
|
+
start: number;
|
|
11
|
+
end: number;
|
|
12
|
+
label: string;
|
|
13
|
+
text: string;
|
|
14
|
+
score: number;
|
|
15
|
+
sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* A PII entity span found by a primary detection layer
|
|
19
|
+
* (regex, NER, legal forms, deny list, ...).
|
|
20
|
+
*/
|
|
21
|
+
type DetectedEntity = EntityBase & {
|
|
22
|
+
source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* An alias mention of a previously detected entity: a
|
|
26
|
+
* defined term ("the Seller") or a propagated bare
|
|
27
|
+
* mention ("Acme" after "Acme Corp.").
|
|
28
|
+
*
|
|
29
|
+
* `corefSourceText` is required by construction, so an
|
|
30
|
+
* alias cannot exist without the link back to its source
|
|
31
|
+
* entity. Placeholder numbering reads it to give the
|
|
32
|
+
* alias the same placeholder as the source. The link
|
|
33
|
+
* travels with the entity instead of living in a
|
|
34
|
+
* side-channel map that a producer could forget to
|
|
35
|
+
* write — or that a later pass could clear.
|
|
36
|
+
*/
|
|
37
|
+
type CorefAliasEntity = EntityBase & {
|
|
38
|
+
source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
|
|
39
|
+
corefSourceText: string;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* A detected PII entity span in the source text.
|
|
43
|
+
* Every detection layer produces these.
|
|
44
|
+
*/
|
|
45
|
+
type Entity = DetectedEntity | CorefAliasEntity;
|
|
46
|
+
/**
|
|
47
|
+
* Entity after human review. Extends the base Entity
|
|
48
|
+
* with a review decision.
|
|
49
|
+
*/
|
|
50
|
+
type ReviewDecision = "confirmed" | "rejected" | "relabeled";
|
|
51
|
+
type ReviewedEntity = Entity & {
|
|
52
|
+
decision?: ReviewDecision;
|
|
53
|
+
originalLabel?: string;
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* A single entry in the workspace-scoped gazetteer
|
|
57
|
+
* (deny list). Persisted in IndexedDB.
|
|
58
|
+
*/
|
|
59
|
+
type GazetteerEntry = {
|
|
60
|
+
id: string;
|
|
61
|
+
canonical: string;
|
|
62
|
+
label: string;
|
|
63
|
+
variants: string[];
|
|
64
|
+
workspaceId: string;
|
|
65
|
+
createdAt: number;
|
|
66
|
+
source: "manual" | "confirmed-from-model";
|
|
67
|
+
};
|
|
68
|
+
/** Extraction strategy — closed discriminated union. */
|
|
69
|
+
type TriggerStrategy = {
|
|
70
|
+
type: "to-next-comma";
|
|
71
|
+
/**
|
|
72
|
+
* Optional list of lowercase keywords that terminate
|
|
73
|
+
* the value scan, in addition to commas/newlines. Useful
|
|
74
|
+
* for triggers like court names that may continue past
|
|
75
|
+
* a missing comma into adjacent clause text ("Městským
|
|
76
|
+
* soudem v Praze dne 1. 1. 2020"); listing `"dne"` here
|
|
77
|
+
* stops the scan at the date boundary. Matched on a
|
|
78
|
+
* word-boundary, case-insensitive.
|
|
79
|
+
*/
|
|
80
|
+
stopWords?: string[];
|
|
81
|
+
/**
|
|
82
|
+
* Hard cap on the captured span length, in characters,
|
|
83
|
+
* regardless of where the next comma / stop char sits.
|
|
84
|
+
* Use for triggers that label short formulaic phrases
|
|
85
|
+
* ("State of Delaware") and must not absorb the rest
|
|
86
|
+
* of a long forum-selection clause when the comma is
|
|
87
|
+
* sentences away. Falls back to the default 100-char
|
|
88
|
+
* fallback when omitted.
|
|
89
|
+
*/
|
|
90
|
+
maxLength?: number;
|
|
91
|
+
} | {
|
|
92
|
+
type: "to-end-of-line";
|
|
93
|
+
} | {
|
|
94
|
+
type: "n-words";
|
|
95
|
+
count: number;
|
|
96
|
+
} | {
|
|
97
|
+
type: "company-id-value";
|
|
98
|
+
} | {
|
|
99
|
+
type: "address";
|
|
100
|
+
maxChars?: number;
|
|
101
|
+
} | {
|
|
102
|
+
/**
|
|
103
|
+
* Extract the first regex match in the value text.
|
|
104
|
+
* Useful for shape-bounded values that follow a
|
|
105
|
+
* label on the same line as other fields, where
|
|
106
|
+
* `to-end-of-line` would over-capture. The pattern
|
|
107
|
+
* is anchored to the start of the (already
|
|
108
|
+
* leading-whitespace-stripped) value, so use
|
|
109
|
+
* `(?:.*?)` prefix only when intentional.
|
|
110
|
+
*/
|
|
111
|
+
type: "match-pattern";
|
|
112
|
+
pattern: string;
|
|
113
|
+
flags?: string;
|
|
114
|
+
};
|
|
115
|
+
/** Validation rules — closed discriminated union. */
|
|
116
|
+
type TriggerValidation = {
|
|
117
|
+
type: "starts-uppercase";
|
|
118
|
+
} | {
|
|
119
|
+
type: "min-length";
|
|
120
|
+
min: number;
|
|
121
|
+
} | {
|
|
122
|
+
type: "max-length";
|
|
123
|
+
max: number;
|
|
124
|
+
} | {
|
|
125
|
+
type: "no-digits";
|
|
126
|
+
} | {
|
|
127
|
+
type: "has-digits";
|
|
128
|
+
} | {
|
|
129
|
+
type: "matches-pattern";
|
|
130
|
+
pattern: string;
|
|
131
|
+
flags?: string;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Run a named stdnum validator (checksum + length)
|
|
135
|
+
* against the captured value. Keeps the trigger
|
|
136
|
+
* path symmetrical with the formatted-regex
|
|
137
|
+
* detectors so e.g. `CPF nº 00000000000` does not
|
|
138
|
+
* survive as a tax-ID entity.
|
|
139
|
+
*/
|
|
140
|
+
| {
|
|
141
|
+
type: "valid-id";
|
|
142
|
+
validator: ValidIdValidator;
|
|
143
|
+
};
|
|
144
|
+
/** Built-in stdnum validators that can be referenced
|
|
145
|
+
* by `valid-id` validations. */
|
|
146
|
+
type ValidIdValidator = "br.cpf" | "br.cnpj" | "us.rtn";
|
|
147
|
+
/** Auto-generated trigger variants — closed set. */
|
|
148
|
+
type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
|
|
149
|
+
/** V2 trigger config entry (JSON shape). */
|
|
150
|
+
type TriggerGroupConfig = {
|
|
151
|
+
id?: string;
|
|
152
|
+
triggers: string[];
|
|
153
|
+
label: string;
|
|
154
|
+
strategy: TriggerStrategy;
|
|
155
|
+
extensions?: TriggerExtension[];
|
|
156
|
+
validations?: TriggerValidation[];
|
|
157
|
+
/** When true, include the trigger text in the
|
|
158
|
+
* entity span (e.g., court names). */
|
|
159
|
+
includeTrigger?: boolean;
|
|
160
|
+
};
|
|
161
|
+
/** Compiled validation with pre-built regex. */
|
|
162
|
+
type CompiledValidation = {
|
|
163
|
+
type: "starts-uppercase";
|
|
164
|
+
re: RegExp;
|
|
165
|
+
} | {
|
|
166
|
+
type: "min-length";
|
|
167
|
+
min: number;
|
|
168
|
+
} | {
|
|
169
|
+
type: "max-length";
|
|
170
|
+
max: number;
|
|
171
|
+
} | {
|
|
172
|
+
type: "no-digits";
|
|
173
|
+
re: RegExp;
|
|
174
|
+
} | {
|
|
175
|
+
type: "has-digits";
|
|
176
|
+
re: RegExp;
|
|
177
|
+
} | {
|
|
178
|
+
type: "matches-pattern";
|
|
179
|
+
re: RegExp;
|
|
180
|
+
} | {
|
|
181
|
+
type: "valid-id";
|
|
182
|
+
validator: ValidIdValidator;
|
|
183
|
+
check: (value: string) => boolean;
|
|
184
|
+
};
|
|
185
|
+
/**
|
|
186
|
+
* Runtime rule — one per trigger string after
|
|
187
|
+
* expansion. Fed to the Aho-Corasick automaton.
|
|
188
|
+
*/
|
|
189
|
+
type TriggerRule = {
|
|
190
|
+
trigger: string;
|
|
191
|
+
label: string;
|
|
192
|
+
strategy: TriggerStrategy;
|
|
193
|
+
validations: CompiledValidation[];
|
|
194
|
+
includeTrigger: boolean;
|
|
195
|
+
};
|
|
196
|
+
/** Per-label operator selection. Key is the entity label. */
|
|
197
|
+
type OperatorConfig = {
|
|
198
|
+
/** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
|
|
199
|
+
redactString: string;
|
|
200
|
+
};
|
|
201
|
+
/** Whether an operator produces a reversible redaction entry. */
|
|
202
|
+
type OperatorReversibility = "reversible" | "irreversible";
|
|
203
|
+
type AnonymisationOperator = {
|
|
204
|
+
type: OperatorType;
|
|
205
|
+
reversibility: OperatorReversibility;
|
|
206
|
+
/**
|
|
207
|
+
* Apply the operator to a single entity occurrence.
|
|
208
|
+
* Returns the replacement string to embed in the document.
|
|
209
|
+
*/
|
|
210
|
+
apply: (text: string, label: string, placeholder: string, redactString: string) => string;
|
|
211
|
+
};
|
|
212
|
+
/**
|
|
213
|
+
* Redacted document output with stable entity mapping.
|
|
214
|
+
*/
|
|
215
|
+
type RedactionResult = {
|
|
216
|
+
redactedText: string;
|
|
217
|
+
/**
|
|
218
|
+
* Maps placeholder to original text. Only populated for
|
|
219
|
+
* reversible operators (replace). Empty for redact.
|
|
220
|
+
*/
|
|
221
|
+
redactionMap: Map<string, string>; /** Maps placeholder to the operator that produced it. */
|
|
222
|
+
operatorMap: Map<string, OperatorType>;
|
|
223
|
+
entityCount: number;
|
|
224
|
+
};
|
|
225
|
+
/**
|
|
226
|
+
* Configuration for the detection pipeline.
|
|
227
|
+
*/
|
|
228
|
+
type DenyListCategory = "Names" | "Places" | "Addresses" | "Courts" | "Financial" | "Government" | "Healthcare" | "Education" | "Political" | "Organizations" | "International";
|
|
229
|
+
/**
|
|
230
|
+
* Metadata for a single dictionary entry in the
|
|
231
|
+
* deny-list system. Mirrors the shape from
|
|
232
|
+
* the anonymize-data package so consumers can pass
|
|
233
|
+
* pre-loaded data without a runtime dependency.
|
|
234
|
+
*/
|
|
235
|
+
type DictionaryMeta = {
|
|
236
|
+
label: string;
|
|
237
|
+
category: DenyListCategory;
|
|
238
|
+
country: string | null;
|
|
239
|
+
};
|
|
240
|
+
/**
|
|
241
|
+
* Caller-supplied exact terms for deny-list matching.
|
|
242
|
+
* These entries are merged with the published deny-list
|
|
243
|
+
* dictionaries when `enableDenyList` is enabled.
|
|
244
|
+
*/
|
|
245
|
+
type CustomDenyListEntry = {
|
|
246
|
+
value: string;
|
|
247
|
+
label: string;
|
|
248
|
+
variants?: readonly string[];
|
|
249
|
+
};
|
|
250
|
+
/**
|
|
251
|
+
* Caller-supplied regex detector. The pattern is passed
|
|
252
|
+
* to the underlying text-search regex engine, so use its
|
|
253
|
+
* supported regex syntax. Inline flags such as `(?i)` are
|
|
254
|
+
* accepted when supported by that engine.
|
|
255
|
+
*/
|
|
256
|
+
type CustomRegexPattern = {
|
|
257
|
+
pattern: string;
|
|
258
|
+
label: string;
|
|
259
|
+
score?: number;
|
|
260
|
+
preparedArtifactPolicy?: "include" | "omit";
|
|
261
|
+
};
|
|
262
|
+
/**
|
|
263
|
+
* Pre-loaded dictionary data for dependency injection.
|
|
264
|
+
* Consumers that want name/city/deny-list detection
|
|
265
|
+
* load dictionaries themselves (e.g. from the
|
|
266
|
+
* anonymize-data package) and pass them here; the
|
|
267
|
+
* anonymize package has zero cross-package imports.
|
|
268
|
+
*
|
|
269
|
+
* All fields are optional. When a field is absent,
|
|
270
|
+
* the corresponding detection path is skipped (same
|
|
271
|
+
* behavior as when no dictionaries are available).
|
|
272
|
+
*/
|
|
273
|
+
type Dictionaries = {
|
|
274
|
+
/**
|
|
275
|
+
* First names per language code (e.g., "cs", "de").
|
|
276
|
+
* Merged with legacy config names at init time.
|
|
277
|
+
*/
|
|
278
|
+
firstNames?: Readonly<Record<string, readonly string[]>>;
|
|
279
|
+
/**
|
|
280
|
+
* Surnames per language code.
|
|
281
|
+
* Merged with legacy config names at init time.
|
|
282
|
+
*/
|
|
283
|
+
surnames?: Readonly<Record<string, readonly string[]>>;
|
|
284
|
+
/**
|
|
285
|
+
* Non-Western name tokens per locale code
|
|
286
|
+
* (e.g., "in", "ar", "ja-latn", "ko", "zh-latn",
|
|
287
|
+
* "th", "vi", "fil", "id"). Merged with bundled
|
|
288
|
+
* names-nw-*.json data at init time.
|
|
289
|
+
*/
|
|
290
|
+
nonWesternNames?: Readonly<Record<string, readonly string[]>>;
|
|
291
|
+
/**
|
|
292
|
+
* Pre-loaded deny-list dictionaries keyed by
|
|
293
|
+
* dictionary ID (e.g., "courts/CZ", "banks/DE").
|
|
294
|
+
* Each value is the array of terms for that
|
|
295
|
+
* dictionary.
|
|
296
|
+
*/
|
|
297
|
+
denyList?: Readonly<Record<string, readonly string[]>>;
|
|
298
|
+
/**
|
|
299
|
+
* Metadata per dictionary ID. Required when
|
|
300
|
+
* `denyList` is provided so the pipeline knows
|
|
301
|
+
* labels, categories, and country filters.
|
|
302
|
+
*/
|
|
303
|
+
denyListMeta?: Readonly<Record<string, DictionaryMeta>>;
|
|
304
|
+
/**
|
|
305
|
+
* Pre-loaded city names, already merged across
|
|
306
|
+
* all desired countries.
|
|
307
|
+
*
|
|
308
|
+
* Prefer `citiesByCountry` when callers also pass
|
|
309
|
+
* `denyListCountries` / `denyListRegions`; merged
|
|
310
|
+
* city arrays cannot be scoped after injection.
|
|
311
|
+
*/
|
|
312
|
+
cities?: readonly string[];
|
|
313
|
+
/**
|
|
314
|
+
* Pre-loaded city names keyed by ISO 3166-1 alpha-2
|
|
315
|
+
* country code. When provided, the deny-list builder
|
|
316
|
+
* applies `denyListCountries` / `denyListRegions`
|
|
317
|
+
* before adding city patterns to the search automaton.
|
|
318
|
+
*/
|
|
319
|
+
citiesByCountry?: Readonly<Record<string, readonly string[]>>;
|
|
320
|
+
};
|
|
321
|
+
type PipelineConfig = {
|
|
322
|
+
threshold: number;
|
|
323
|
+
enableTriggerPhrases: boolean;
|
|
324
|
+
enableRegex: boolean;
|
|
325
|
+
/**
|
|
326
|
+
* Expected content language codes. When present, these
|
|
327
|
+
* derive default dictionary scopes for name corpus and
|
|
328
|
+
* deny-list matching unless the lower-level scope fields
|
|
329
|
+
* below are set explicitly.
|
|
330
|
+
*/
|
|
331
|
+
languages?: string[];
|
|
332
|
+
/**
|
|
333
|
+
* Convenience form for single-language documents. Ignored
|
|
334
|
+
* when `languages` is also provided.
|
|
335
|
+
*/
|
|
336
|
+
language?: string;
|
|
337
|
+
/**
|
|
338
|
+
* Enables legal-form organization detection.
|
|
339
|
+
* Required for typed callers; legacy untyped
|
|
340
|
+
* callers that omit this field are treated as
|
|
341
|
+
* enabled at runtime for backward compatibility.
|
|
342
|
+
*/
|
|
343
|
+
enableLegalForms: boolean;
|
|
344
|
+
/**
|
|
345
|
+
* Enables first-name/surname/title corpus matching.
|
|
346
|
+
* When deny-list mode is enabled, this also controls
|
|
347
|
+
* whether name-corpus entries are injected into the
|
|
348
|
+
* deny-list search automaton.
|
|
349
|
+
*/
|
|
350
|
+
enableNameCorpus: boolean;
|
|
351
|
+
/**
|
|
352
|
+
* Optional language scope for first-name/surname
|
|
353
|
+
* dictionaries, using the keys present in
|
|
354
|
+
* `dictionaries.firstNames` / `dictionaries.surnames`
|
|
355
|
+
* (for example `["en", "de"]`). When omitted, all
|
|
356
|
+
* injected name languages are used for backward
|
|
357
|
+
* compatibility.
|
|
358
|
+
*/
|
|
359
|
+
nameCorpusLanguages?: string[];
|
|
360
|
+
enableDenyList: boolean;
|
|
361
|
+
denyListCountries?: string[];
|
|
362
|
+
denyListRegions?: string[];
|
|
363
|
+
denyListExcludeCategories?: string[];
|
|
364
|
+
/**
|
|
365
|
+
* Caller-owned exact terms to match through the
|
|
366
|
+
* deny-list layer. Requires `enableDenyList: true`.
|
|
367
|
+
*/
|
|
368
|
+
customDenyList?: readonly CustomDenyListEntry[];
|
|
369
|
+
/**
|
|
370
|
+
* Caller-owned regex detectors. Requires
|
|
371
|
+
* `enableRegex: true`.
|
|
372
|
+
*/
|
|
373
|
+
customRegexes?: readonly CustomRegexPattern[];
|
|
374
|
+
enableGazetteer: boolean;
|
|
375
|
+
/**
|
|
376
|
+
* Detect country names (ISO 3166-1 names, curated
|
|
377
|
+
* aliases, alpha-3 codes). Defaults to true. Names
|
|
378
|
+
* span all manifest languages plus widely-used
|
|
379
|
+
* additions (Dutch, Russian, Chinese, Arabic, etc.).
|
|
380
|
+
*/
|
|
381
|
+
enableCountries?: boolean;
|
|
382
|
+
enableNer: boolean;
|
|
383
|
+
enableConfidenceBoost: boolean;
|
|
384
|
+
enableCoreference: boolean;
|
|
385
|
+
enableZoneClassification?: boolean;
|
|
386
|
+
enableHotwordRules?: boolean;
|
|
387
|
+
/**
|
|
388
|
+
* Requested output labels. An empty array means
|
|
389
|
+
* "do not filter by label" for deterministic
|
|
390
|
+
* detectors; NER falls back to DEFAULT_ENTITY_LABELS.
|
|
391
|
+
*/
|
|
392
|
+
labels: string[];
|
|
393
|
+
workspaceId: string;
|
|
394
|
+
/**
|
|
395
|
+
* Pre-loaded dictionary data for name, deny-list,
|
|
396
|
+
* and city detection. When omitted, dictionary-based
|
|
397
|
+
* detection paths are skipped. Consumers load from
|
|
398
|
+
* the anonymize-data package and pass the data here.
|
|
399
|
+
*/
|
|
400
|
+
dictionaries?: Dictionaries;
|
|
401
|
+
};
|
|
402
|
+
//#endregion
|
|
403
|
+
//#region src/detectors/regex.d.ts
|
|
404
|
+
type RegexMeta = {
|
|
405
|
+
label: string;
|
|
406
|
+
score: number;
|
|
407
|
+
sourceDetail?: Entity["sourceDetail"];
|
|
408
|
+
minByteLength?: number; /** Post-match stdnum validator for confirmation. */
|
|
409
|
+
validator?: Validator;
|
|
410
|
+
validatorId?: string; /** Extract the identifier portion when context is part of the regex span. */
|
|
411
|
+
validatorInput?: (text: string) => string;
|
|
412
|
+
validatorInputKind?: "digits-only" | "crypto-wallet-candidate";
|
|
413
|
+
};
|
|
414
|
+
type DateMonthData = Record<string, string[]>;
|
|
415
|
+
type YearWordData = Record<string, string[]>;
|
|
416
|
+
type MonetaryData = {
|
|
417
|
+
currencies: {
|
|
418
|
+
codes: string[];
|
|
419
|
+
symbols: string[];
|
|
420
|
+
local_names: string[];
|
|
421
|
+
};
|
|
422
|
+
amount_words: {
|
|
423
|
+
written_amount_patterns: Array<{
|
|
424
|
+
keywords: string[];
|
|
425
|
+
}>;
|
|
426
|
+
magnitude_suffixes: Array<{
|
|
427
|
+
words: string[];
|
|
428
|
+
abbreviations_case_insensitive: string[];
|
|
429
|
+
abbreviations_case_sensitive: string[];
|
|
430
|
+
}>;
|
|
431
|
+
share_quantity_terms: Array<{
|
|
432
|
+
modifiers: string[];
|
|
433
|
+
nouns: string[];
|
|
434
|
+
}>;
|
|
435
|
+
};
|
|
436
|
+
};
|
|
437
|
+
//#endregion
|
|
438
|
+
//#region src/context.d.ts
|
|
439
|
+
/**
|
|
440
|
+
* Compiled RegExp pattern used for coreference
|
|
441
|
+
* definition extraction.
|
|
442
|
+
*/
|
|
443
|
+
type DefinitionPattern = {
|
|
444
|
+
pattern: RegExp;
|
|
445
|
+
};
|
|
446
|
+
/**
|
|
447
|
+
* Cached data for the name corpus detector.
|
|
448
|
+
* Populated by initNameCorpus; consumed by
|
|
449
|
+
* detectNameCorpus and deny-list AC integration.
|
|
450
|
+
*/
|
|
451
|
+
type NameCorpusData = {
|
|
452
|
+
firstNames: ReadonlySet<string>;
|
|
453
|
+
surnames: ReadonlySet<string>;
|
|
454
|
+
titleTokens: ReadonlySet<string>;
|
|
455
|
+
/** Abbreviation-style titles whose trailing dot is
|
|
456
|
+
* part of the title, not a sentence boundary.
|
|
457
|
+
* Contains the lowercase, dot-stripped form
|
|
458
|
+
* (e.g., "dr", "smt", "atty"). */
|
|
459
|
+
titleAbbreviations: ReadonlySet<string>;
|
|
460
|
+
excludedWords: ReadonlySet<string>;
|
|
461
|
+
/** Lowercased common English words. A name chain whose
|
|
462
|
+
* every token is a common word (e.g. "Loan Documents",
|
|
463
|
+
* where "Loan" coincides with a Vietnamese given name)
|
|
464
|
+
* is treated as a common-word phrase, not a person. */
|
|
465
|
+
commonWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
|
|
466
|
+
nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
|
|
467
|
+
excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
|
|
468
|
+
firstNamesList: readonly string[];
|
|
469
|
+
surnamesList: readonly string[];
|
|
470
|
+
titlesList: readonly string[];
|
|
471
|
+
excludedList: readonly string[];
|
|
472
|
+
nonWesternNamesList: readonly string[];
|
|
473
|
+
excludedAllCapsList: readonly string[];
|
|
474
|
+
};
|
|
475
|
+
/**
|
|
476
|
+
* All cached state for a single pipeline run (or
|
|
477
|
+
* sequence of runs sharing the same config). Replacing
|
|
478
|
+
* module-level singletons with this object enables
|
|
479
|
+
* concurrent pipelines with different configs and
|
|
480
|
+
* simplifies testing.
|
|
481
|
+
*
|
|
482
|
+
* Each field starts null and is populated lazily on
|
|
483
|
+
* first use by the corresponding loader function.
|
|
484
|
+
*/
|
|
485
|
+
type PipelineContext = {
|
|
486
|
+
search: UnifiedSearchInstance | null;
|
|
487
|
+
searchKey: string;
|
|
488
|
+
searchPromise: Promise<UnifiedSearchInstance> | null;
|
|
489
|
+
nativePipelinePackage: Uint8Array | null;
|
|
490
|
+
nativePipelinePackageKey: string;
|
|
491
|
+
nativePipelinePackagePromise: Promise<Uint8Array> | null;
|
|
492
|
+
nameCorpus: NameCorpusData | null;
|
|
493
|
+
nameCorpusKey: string;
|
|
494
|
+
nameCorpusPromise: Promise<void> | null;
|
|
495
|
+
stopwords: ReadonlySet<string> | null;
|
|
496
|
+
stopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
497
|
+
allowList: ReadonlySet<string> | null;
|
|
498
|
+
allowListPromise: Promise<ReadonlySet<string>> | null;
|
|
499
|
+
personStopwords: ReadonlySet<string> | null;
|
|
500
|
+
personStopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
501
|
+
definedTermHeads: ReadonlySet<string> | null;
|
|
502
|
+
definedTermHeadsPromise: Promise<ReadonlySet<string>> | null;
|
|
503
|
+
addressStopwords: ReadonlySet<string> | null;
|
|
504
|
+
addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
|
|
505
|
+
firstNameExclusions: ReadonlySet<string> | null;
|
|
506
|
+
firstNameExclusionCorpusLen: number;
|
|
507
|
+
genericRoles: ReadonlySet<string> | null;
|
|
508
|
+
genericRolesPromise: Promise<ReadonlySet<string>> | null;
|
|
509
|
+
corefPatterns: DefinitionPattern[] | null;
|
|
510
|
+
corefPatternsKey: string;
|
|
511
|
+
corefPatternsPromise: Promise<DefinitionPattern[]> | null;
|
|
512
|
+
corefLoadAttempted: boolean;
|
|
513
|
+
roleStopSet: ReadonlySet<string> | null;
|
|
514
|
+
roleStopSetPromise: Promise<ReadonlySet<string>> | null;
|
|
515
|
+
zoneHeadingPatterns: RegExp[] | null;
|
|
516
|
+
zoneSigningPatterns: RegExp[] | null;
|
|
517
|
+
zoneInitPromise: Promise<void> | null;
|
|
518
|
+
};
|
|
519
|
+
//#endregion
|
|
520
|
+
//#region src/detectors/deny-list.d.ts
|
|
521
|
+
type DenyListFilterData = {
|
|
522
|
+
stopwords: string[];
|
|
523
|
+
allowList: string[];
|
|
524
|
+
personStopwords: string[];
|
|
525
|
+
personTrailingNouns: string[];
|
|
526
|
+
addressStopwords: string[];
|
|
527
|
+
addressJurisdictionPrefixes: string[];
|
|
528
|
+
streetTypes: string[];
|
|
529
|
+
addressComponentTerms: string[];
|
|
530
|
+
ambiguousStreetTypeTerms: string[];
|
|
531
|
+
firstNames: string[];
|
|
532
|
+
genericRoles: string[];
|
|
533
|
+
numberAbbrevPrefixes: string[];
|
|
534
|
+
sentenceStarters: string[];
|
|
535
|
+
trailingAddressWordExclusions: string[];
|
|
536
|
+
documentHeadingWords: string[];
|
|
537
|
+
documentHeadingOrdinalMarkers: string[];
|
|
538
|
+
definedTermCues: string[];
|
|
539
|
+
signingPlaceGuards: DenyListSigningPlaceGuardData[];
|
|
540
|
+
};
|
|
541
|
+
type DenyListSigningPlaceGuardData = {
|
|
542
|
+
prefixPhrases: string[];
|
|
543
|
+
suffixPhrases: string[];
|
|
544
|
+
};
|
|
545
|
+
/**
|
|
546
|
+
* Source tag for each pattern in the automaton.
|
|
547
|
+
* "deny-list" = standard deny list entry
|
|
548
|
+
* "city" = city dictionary entry
|
|
549
|
+
* "custom-deny-list" = caller-owned exact term
|
|
550
|
+
* "first-name" = name corpus first name
|
|
551
|
+
* "surname" = name corpus surname
|
|
552
|
+
* "title" = academic/professional title
|
|
553
|
+
*/
|
|
554
|
+
type PatternSource = "deny-list" | "city" | "custom-deny-list" | "first-name" | "surname" | "title";
|
|
555
|
+
type PatternLabels = string | string[];
|
|
556
|
+
type PatternSources = PatternSource | PatternSource[];
|
|
557
|
+
/**
|
|
558
|
+
* Pre-built deny list data. Constructed once by
|
|
559
|
+
* `buildDenyList`, reused across `processDenyListMatches`
|
|
560
|
+
* calls. Contains PatternEntry[] for the unified builder
|
|
561
|
+
* plus parallel label/source arrays for post-processing.
|
|
562
|
+
*/
|
|
563
|
+
type DenyListData = {
|
|
564
|
+
/**
|
|
565
|
+
* Maps pattern index → entity labels (plural).
|
|
566
|
+
* Same pattern can have multiple labels when it
|
|
567
|
+
* appears in multiple dictionaries (e.g., "Denver"
|
|
568
|
+
* is both a person name and a city name).
|
|
569
|
+
*/
|
|
570
|
+
labels: PatternLabels[]; /** Maps pattern index → labels contributed by custom entries. */
|
|
571
|
+
customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
|
|
572
|
+
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
573
|
+
sources: PatternSources[];
|
|
574
|
+
filters: DenyListFilterData;
|
|
575
|
+
};
|
|
576
|
+
//#endregion
|
|
577
|
+
//#region src/detectors/address-seeds.d.ts
|
|
578
|
+
type AddressSeedData = {
|
|
579
|
+
boundary_words: string[];
|
|
580
|
+
br_cep_cue_words: string[];
|
|
581
|
+
unit_abbreviations: string[];
|
|
582
|
+
};
|
|
583
|
+
//#endregion
|
|
584
|
+
//#region src/detectors/countries.d.ts
|
|
585
|
+
/**
|
|
586
|
+
* Pre-built country patterns + parallel label/source
|
|
587
|
+
* metadata. Constructed once and reused across pipeline
|
|
588
|
+
* runs.
|
|
589
|
+
*/
|
|
590
|
+
type CountryData = {
|
|
591
|
+
/** Maps local pattern index to entity label. Always "country". */labels: string[];
|
|
592
|
+
/**
|
|
593
|
+
* Maps local pattern index to the alpha-2 ISO code the
|
|
594
|
+
* pattern resolves to. Used for downstream coreference /
|
|
595
|
+
* placeholder grouping.
|
|
596
|
+
*/
|
|
597
|
+
isoCodes: string[]; /** Maps local pattern index to pattern variant kind. */
|
|
598
|
+
variants: CountryVariant[];
|
|
599
|
+
};
|
|
600
|
+
type CountryVariant = "name" | "alias" | "alpha3" | "alpha2";
|
|
601
|
+
//#endregion
|
|
602
|
+
//#region src/filters/confidence-boost.d.ts
|
|
603
|
+
type AddressContextData = {
|
|
604
|
+
address_prepositions: string[];
|
|
605
|
+
temporal_prepositions: string[];
|
|
606
|
+
street_abbreviations: string[];
|
|
607
|
+
bare_house_stopwords: string[];
|
|
608
|
+
};
|
|
609
|
+
//#endregion
|
|
610
|
+
//#region src/build-unified-search.d.ts
|
|
611
|
+
type PatternSlice = {
|
|
612
|
+
start: number;
|
|
613
|
+
end: number;
|
|
614
|
+
};
|
|
615
|
+
type NativeSearchPatternKind = "literal" | "literal-with-options" | "regex" | "fuzzy";
|
|
616
|
+
type NativeSearchPattern = {
|
|
617
|
+
kind: NativeSearchPatternKind;
|
|
618
|
+
pattern: string;
|
|
619
|
+
distance?: number;
|
|
620
|
+
case_insensitive?: boolean;
|
|
621
|
+
whole_words?: boolean;
|
|
622
|
+
lazy?: boolean;
|
|
623
|
+
prefilter_any?: string[];
|
|
624
|
+
prefilter_case_insensitive?: boolean;
|
|
625
|
+
prefilter_regex?: string;
|
|
626
|
+
prefilter_window_bytes?: number;
|
|
627
|
+
prepared_artifact_policy?: "include" | "omit";
|
|
628
|
+
};
|
|
629
|
+
type NativeSearchOptions = {
|
|
630
|
+
literal_case_insensitive?: boolean;
|
|
631
|
+
literal_whole_words?: boolean;
|
|
632
|
+
regex_whole_words?: boolean;
|
|
633
|
+
regex_overlap_all?: boolean;
|
|
634
|
+
regex_artifact_policy?: "include" | "omit";
|
|
635
|
+
fuzzy_case_insensitive?: boolean;
|
|
636
|
+
fuzzy_whole_words?: boolean;
|
|
637
|
+
fuzzy_normalize_diacritics?: boolean;
|
|
638
|
+
};
|
|
639
|
+
type NativeRegexMatchMeta = {
|
|
640
|
+
label: string;
|
|
641
|
+
score: number;
|
|
642
|
+
source_detail?: string;
|
|
643
|
+
requires_validation?: boolean;
|
|
644
|
+
validator_id?: string;
|
|
645
|
+
validator_input?: string;
|
|
646
|
+
min_byte_length?: number;
|
|
647
|
+
};
|
|
648
|
+
type NativeDenyListFilterData = {
|
|
649
|
+
stopwords: string[];
|
|
650
|
+
allow_list: string[];
|
|
651
|
+
person_stopwords: string[];
|
|
652
|
+
person_trailing_nouns: string[];
|
|
653
|
+
address_stopwords: string[];
|
|
654
|
+
address_jurisdiction_prefixes: string[];
|
|
655
|
+
street_types: string[];
|
|
656
|
+
address_component_terms: string[];
|
|
657
|
+
ambiguous_street_type_terms: string[];
|
|
658
|
+
first_names: string[];
|
|
659
|
+
generic_roles: string[];
|
|
660
|
+
number_abbrev_prefixes: string[];
|
|
661
|
+
sentence_starters: string[];
|
|
662
|
+
trailing_address_word_exclusions: string[];
|
|
663
|
+
document_heading_words: string[];
|
|
664
|
+
document_heading_ordinal_markers: string[];
|
|
665
|
+
defined_term_cues: string[];
|
|
666
|
+
signing_place_guards: NativeSigningPlaceGuardData[];
|
|
667
|
+
};
|
|
668
|
+
type NativeSigningPlaceGuardData = {
|
|
669
|
+
prefix_phrases: string[];
|
|
670
|
+
suffix_phrases: string[];
|
|
671
|
+
};
|
|
672
|
+
type NativeDenyListMatchData = {
|
|
673
|
+
labels?: string[][];
|
|
674
|
+
label_table?: string[];
|
|
675
|
+
label_indices?: number[][];
|
|
676
|
+
custom_labels?: string[][];
|
|
677
|
+
custom_label_indices?: number[][];
|
|
678
|
+
originals: string[];
|
|
679
|
+
sources?: string[][];
|
|
680
|
+
source_table?: string[];
|
|
681
|
+
source_indices?: number[][];
|
|
682
|
+
filters?: NativeDenyListFilterData;
|
|
683
|
+
};
|
|
684
|
+
type NativeTriggerStrategy = {
|
|
685
|
+
type: "to-next-comma";
|
|
686
|
+
stop_words?: string[];
|
|
687
|
+
max_length?: number;
|
|
688
|
+
} | {
|
|
689
|
+
type: "to-end-of-line";
|
|
690
|
+
} | {
|
|
691
|
+
type: "n-words";
|
|
692
|
+
count: number;
|
|
693
|
+
} | {
|
|
694
|
+
type: "company-id-value";
|
|
695
|
+
} | {
|
|
696
|
+
type: "address";
|
|
697
|
+
max_chars?: number;
|
|
698
|
+
} | {
|
|
699
|
+
type: "match-pattern";
|
|
700
|
+
pattern: string;
|
|
701
|
+
flags?: string;
|
|
702
|
+
};
|
|
703
|
+
type NativeTriggerValidation = {
|
|
704
|
+
type: "starts-uppercase";
|
|
705
|
+
} | {
|
|
706
|
+
type: "min-length";
|
|
707
|
+
min: number;
|
|
708
|
+
} | {
|
|
709
|
+
type: "max-length";
|
|
710
|
+
max: number;
|
|
711
|
+
} | {
|
|
712
|
+
type: "no-digits";
|
|
713
|
+
} | {
|
|
714
|
+
type: "has-digits";
|
|
715
|
+
} | {
|
|
716
|
+
type: "matches-pattern";
|
|
717
|
+
pattern: string;
|
|
718
|
+
flags?: string;
|
|
719
|
+
} | {
|
|
720
|
+
type: "valid-id";
|
|
721
|
+
validator: string;
|
|
722
|
+
};
|
|
723
|
+
type NativeTriggerRule = {
|
|
724
|
+
trigger: string;
|
|
725
|
+
label: string;
|
|
726
|
+
strategy: NativeTriggerStrategy;
|
|
727
|
+
validations: NativeTriggerValidation[];
|
|
728
|
+
include_trigger: boolean;
|
|
729
|
+
};
|
|
730
|
+
type NativeTriggerData = {
|
|
731
|
+
rules: NativeTriggerRule[];
|
|
732
|
+
address_stop_keywords: string[];
|
|
733
|
+
party_position_terms: string[];
|
|
734
|
+
post_nominals: string[];
|
|
735
|
+
sentence_terminal_currency_terms: string[];
|
|
736
|
+
phone_extension_labels: string[];
|
|
737
|
+
number_markers: string[];
|
|
738
|
+
number_labels: string[];
|
|
739
|
+
};
|
|
740
|
+
type NativeLegalFormData = {
|
|
741
|
+
suffixes: string[];
|
|
742
|
+
normalized_boundary_suffixes: string[];
|
|
743
|
+
normalized_in_name_words: string[];
|
|
744
|
+
normalized_suffix_words: string[];
|
|
745
|
+
role_heads: string[];
|
|
746
|
+
sentence_verb_indicators: string[];
|
|
747
|
+
clause_noun_heads: string[];
|
|
748
|
+
connector_prose_heads: string[];
|
|
749
|
+
structural_single_cap_prefixes: string[];
|
|
750
|
+
leading_clause_phrases: string[];
|
|
751
|
+
leading_clause_direct_prefixes: string[];
|
|
752
|
+
connector_words: string[];
|
|
753
|
+
and_connector_words: string[];
|
|
754
|
+
in_name_prepositions: string[];
|
|
755
|
+
company_suffix_words: string[];
|
|
756
|
+
comma_gated_direct_prefixes: string[];
|
|
757
|
+
};
|
|
758
|
+
type NativeDateData = {
|
|
759
|
+
month_names_by_language: DateMonthData;
|
|
760
|
+
year_words_by_language: YearWordData;
|
|
761
|
+
};
|
|
762
|
+
type NativeMonetaryData = MonetaryData;
|
|
763
|
+
type NativeAddressSeedData = AddressSeedData;
|
|
764
|
+
type NativeAddressContextData = AddressContextData;
|
|
765
|
+
type NativeCoreferencePatternData = {
|
|
766
|
+
pattern: string;
|
|
767
|
+
flags: string;
|
|
768
|
+
};
|
|
769
|
+
type NativeCoreferenceData = {
|
|
770
|
+
definition_patterns: NativeCoreferencePatternData[];
|
|
771
|
+
role_stop_terms: string[];
|
|
772
|
+
legal_form_aliases: string[];
|
|
773
|
+
organization_suffixes: string[];
|
|
774
|
+
organization_determiners: string[];
|
|
775
|
+
};
|
|
776
|
+
type NativeNameCorpusData = {
|
|
777
|
+
first_names: string[];
|
|
778
|
+
surnames: string[];
|
|
779
|
+
title_tokens: string[];
|
|
780
|
+
title_abbreviations: string[];
|
|
781
|
+
excluded_words: string[];
|
|
782
|
+
common_words: string[];
|
|
783
|
+
non_western_names: string[];
|
|
784
|
+
excluded_all_caps: string[];
|
|
785
|
+
ja_suffixes: string[];
|
|
786
|
+
arabic_connectors: string[];
|
|
787
|
+
relation_connectors: string[];
|
|
788
|
+
hyphenated_prefixes: string[];
|
|
789
|
+
cjk_non_person_terms: string[];
|
|
790
|
+
cjk_surname_starters: string[];
|
|
791
|
+
organization_terms: string[];
|
|
792
|
+
};
|
|
793
|
+
type NativeNameCorpusMode = "full" | "supplemental";
|
|
794
|
+
type NativeZonePatternData = {
|
|
795
|
+
pattern: string;
|
|
796
|
+
flags: string;
|
|
797
|
+
};
|
|
798
|
+
type NativeZoneSigningClauseData = {
|
|
799
|
+
prefix: string;
|
|
800
|
+
suffix: string;
|
|
801
|
+
prepositions: string[];
|
|
802
|
+
};
|
|
803
|
+
type NativeZoneData = {
|
|
804
|
+
section_heading_patterns: NativeZonePatternData[];
|
|
805
|
+
signing_clauses: NativeZoneSigningClauseData[];
|
|
806
|
+
};
|
|
807
|
+
type NativeGazetteerData = {
|
|
808
|
+
labels: string[];
|
|
809
|
+
is_fuzzy: boolean[];
|
|
810
|
+
};
|
|
811
|
+
type NativeHotwordRule = {
|
|
812
|
+
hotwords: string[];
|
|
813
|
+
target_labels: string[];
|
|
814
|
+
score_adjustment: number;
|
|
815
|
+
reclassify_to?: string;
|
|
816
|
+
proximity_before: number;
|
|
817
|
+
proximity_after: number;
|
|
818
|
+
};
|
|
819
|
+
type NativeHotwordRuleData = {
|
|
820
|
+
rules: NativeHotwordRule[];
|
|
821
|
+
pattern_rule_indices: number[];
|
|
822
|
+
};
|
|
823
|
+
type NativeSignatureData = {
|
|
824
|
+
labels: string[];
|
|
825
|
+
witness_phrases: string[];
|
|
826
|
+
name_particles: string[];
|
|
827
|
+
post_nominal_suffixes: string[];
|
|
828
|
+
organization_suffixes: string[];
|
|
829
|
+
image_stub_prefixes: string[];
|
|
830
|
+
};
|
|
831
|
+
type NativePreparedSearchConfig = {
|
|
832
|
+
regex_patterns: NativeSearchPattern[];
|
|
833
|
+
custom_regex_patterns: NativeSearchPattern[];
|
|
834
|
+
literal_patterns: NativeSearchPattern[];
|
|
835
|
+
regex_options: NativeSearchOptions;
|
|
836
|
+
custom_regex_options: NativeSearchOptions;
|
|
837
|
+
literal_options: NativeSearchOptions;
|
|
838
|
+
literal_patterns_from_deny_list_data?: boolean;
|
|
839
|
+
allowed_labels: string[];
|
|
840
|
+
threshold: number;
|
|
841
|
+
confidence_boost: boolean;
|
|
842
|
+
slices: {
|
|
843
|
+
regex: PatternSlice;
|
|
844
|
+
custom_regex: PatternSlice;
|
|
845
|
+
legal_forms?: PatternSlice;
|
|
846
|
+
triggers?: PatternSlice;
|
|
847
|
+
deny_list: PatternSlice;
|
|
848
|
+
street_types?: PatternSlice;
|
|
849
|
+
gazetteer: PatternSlice;
|
|
850
|
+
countries: PatternSlice;
|
|
851
|
+
hotwords?: PatternSlice;
|
|
852
|
+
};
|
|
853
|
+
regex_meta: NativeRegexMatchMeta[];
|
|
854
|
+
custom_regex_meta: NativeRegexMatchMeta[];
|
|
855
|
+
deny_list_data?: NativeDenyListMatchData;
|
|
856
|
+
false_positive_filters?: NativeDenyListFilterData;
|
|
857
|
+
gazetteer_data?: NativeGazetteerData;
|
|
858
|
+
country_data?: CountryData;
|
|
859
|
+
hotword_data?: NativeHotwordRuleData;
|
|
860
|
+
trigger_data?: NativeTriggerData;
|
|
861
|
+
legal_form_data?: NativeLegalFormData;
|
|
862
|
+
address_seed_data?: NativeAddressSeedData;
|
|
863
|
+
zone_data?: NativeZoneData;
|
|
864
|
+
address_context_data?: NativeAddressContextData;
|
|
865
|
+
coreference_data?: NativeCoreferenceData;
|
|
866
|
+
name_corpus_data?: NativeNameCorpusData;
|
|
867
|
+
signature_data?: NativeSignatureData;
|
|
868
|
+
name_corpus_mode?: NativeNameCorpusMode;
|
|
869
|
+
date_data?: NativeDateData;
|
|
870
|
+
monetary_data?: NativeMonetaryData;
|
|
871
|
+
};
|
|
872
|
+
type GazetteerData = {
|
|
873
|
+
/** Maps local pattern index to entry label. */labels: string[];
|
|
874
|
+
/**
|
|
875
|
+
* Whether each pattern is fuzzy (distance > 0).
|
|
876
|
+
* Used by the post-processor to assign scores.
|
|
877
|
+
*/
|
|
878
|
+
isFuzzy: boolean[];
|
|
879
|
+
};
|
|
880
|
+
type UnifiedSearchInstance = {
|
|
881
|
+
/** Regex + triggers + legal-forms. */tsRegex: TextSearch; /** Caller-owned custom regexes, isolated for overlap preservation. */
|
|
882
|
+
tsCustomRegex: TextSearch; /** Deny-list + street-types + gazetteer. */
|
|
883
|
+
tsLiterals: TextSearch;
|
|
884
|
+
slices: {
|
|
885
|
+
regex: PatternSlice;
|
|
886
|
+
customRegex: PatternSlice;
|
|
887
|
+
legalForms: PatternSlice;
|
|
888
|
+
triggers: PatternSlice;
|
|
889
|
+
denyList: PatternSlice;
|
|
890
|
+
streetTypes: PatternSlice;
|
|
891
|
+
gazetteer: PatternSlice;
|
|
892
|
+
countries: PatternSlice;
|
|
893
|
+
};
|
|
894
|
+
regexMeta: readonly RegexMeta[];
|
|
895
|
+
customRegexMeta: readonly RegexMeta[];
|
|
896
|
+
triggerRules: readonly TriggerRule[];
|
|
897
|
+
denyListData: DenyListData | null;
|
|
898
|
+
gazetteerData: GazetteerData | null;
|
|
899
|
+
countryData: CountryData | null;
|
|
900
|
+
nativeStaticConfig: NativePreparedSearchConfig;
|
|
901
|
+
};
|
|
902
|
+
//#endregion
|
|
903
|
+
//#region src/native.d.ts
|
|
904
|
+
type NativeBindingOperatorConfig = {
|
|
905
|
+
operators?: Record<string, OperatorType>;
|
|
906
|
+
redactString?: string;
|
|
907
|
+
};
|
|
908
|
+
type NativeDiagnosticsBatchCallback = (diagnosticsJson: string) => void;
|
|
909
|
+
type NativeResultEventCallback = (eventJson: string) => void;
|
|
910
|
+
type NativeBindingRedactionEntry = {
|
|
911
|
+
placeholder: string;
|
|
912
|
+
original: string;
|
|
913
|
+
};
|
|
914
|
+
type NativeBindingOperatorEntry = {
|
|
915
|
+
placeholder: string;
|
|
916
|
+
operator: OperatorType;
|
|
917
|
+
};
|
|
918
|
+
type NativeBindingPipelineEntity = {
|
|
919
|
+
start: number;
|
|
920
|
+
end: number;
|
|
921
|
+
label: string;
|
|
922
|
+
text: string;
|
|
923
|
+
score: number;
|
|
924
|
+
source: string;
|
|
925
|
+
sourceDetail?: string | null;
|
|
926
|
+
};
|
|
927
|
+
type NativeBindingRedactionResult = {
|
|
928
|
+
redactedText: string;
|
|
929
|
+
redactionMap: NativeBindingRedactionEntry[];
|
|
930
|
+
operatorMap: NativeBindingOperatorEntry[];
|
|
931
|
+
entityCount: number;
|
|
932
|
+
};
|
|
933
|
+
type NativeBindingStaticRedactionResult = {
|
|
934
|
+
resolvedEntities: NativeBindingPipelineEntity[];
|
|
935
|
+
redaction: NativeBindingRedactionResult;
|
|
936
|
+
};
|
|
937
|
+
type NativePreparedSearchBinding = {
|
|
938
|
+
prepareDiagnosticsJson?: () => string;
|
|
939
|
+
warmLazyRegex?: () => void;
|
|
940
|
+
warm_lazy_regex?: () => void;
|
|
941
|
+
warmLazyRegexDiagnosticsJson?: () => string;
|
|
942
|
+
warm_lazy_regex_diagnostics_json?: () => string;
|
|
943
|
+
redactStaticEntities: (fullText: string, operators?: NativeBindingOperatorConfig) => NativeBindingStaticRedactionResult;
|
|
944
|
+
redactStaticEntitiesJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
945
|
+
redactStaticEntitiesResultStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onEvent: NativeResultEventCallback) => string;
|
|
946
|
+
redactStaticEntitiesDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
947
|
+
redactStaticEntitiesDiagnosticsStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onBatch: NativeDiagnosticsBatchCallback) => string;
|
|
948
|
+
redactStaticEntitiesSummaryDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
949
|
+
};
|
|
950
|
+
type NativeAnonymizeBinding = {
|
|
951
|
+
normalizeForSearch: (text: string) => string;
|
|
952
|
+
nativePackageVersion: () => string;
|
|
953
|
+
NativePreparedSearch: {
|
|
954
|
+
fromConfigJsonBytes: (configJson: Uint8Array) => NativePreparedSearchBinding;
|
|
955
|
+
fromPreparedPackageBytes: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
956
|
+
fromPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
957
|
+
fromTrustedPreparedPackageBytes?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
958
|
+
fromTrustedPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
959
|
+
};
|
|
960
|
+
prepareStaticSearchPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
961
|
+
prepareStaticSearchCompressedPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
962
|
+
};
|
|
963
|
+
type NativeOperatorConfig = {
|
|
964
|
+
operators?: Record<string, OperatorType>;
|
|
965
|
+
redactString?: string;
|
|
966
|
+
};
|
|
967
|
+
type NativePipelineEntity = {
|
|
968
|
+
start: number;
|
|
969
|
+
end: number;
|
|
970
|
+
label: string;
|
|
971
|
+
text: string;
|
|
972
|
+
score: number;
|
|
973
|
+
source: string;
|
|
974
|
+
sourceDetail?: string;
|
|
975
|
+
};
|
|
976
|
+
type NativeRedactionResult = {
|
|
977
|
+
redactedText: string;
|
|
978
|
+
redactionMap: Map<string, string>;
|
|
979
|
+
operatorMap: Map<string, OperatorType>;
|
|
980
|
+
entityCount: number;
|
|
981
|
+
};
|
|
982
|
+
type NativeStaticRedactionResult = {
|
|
983
|
+
resolvedEntities: NativePipelineEntity[];
|
|
984
|
+
redaction: NativeRedactionResult;
|
|
985
|
+
};
|
|
986
|
+
type NativeSearchPackageOptions = {
|
|
987
|
+
binding: NativeAnonymizeBinding;
|
|
988
|
+
config: NativePreparedSearchConfig;
|
|
989
|
+
compressed?: boolean;
|
|
990
|
+
};
|
|
991
|
+
type NativeSearchPackageInput = NativePreparedSearchConfig | string | Uint8Array;
|
|
992
|
+
type SharedNativeSearchPackageOptions = {
|
|
993
|
+
binding: NativeAnonymizeBinding;
|
|
994
|
+
config: NativeSearchPackageInput;
|
|
995
|
+
compressed?: boolean;
|
|
996
|
+
};
|
|
997
|
+
type SharedNativePreparedPackageOptions = {
|
|
998
|
+
binding: NativeAnonymizeBinding;
|
|
999
|
+
packageBytes: Uint8Array;
|
|
1000
|
+
};
|
|
1001
|
+
type SharedNativeRedactTextJsonOptions = {
|
|
1002
|
+
binding: NativeAnonymizeBinding;
|
|
1003
|
+
config: NativeSearchPackageInput;
|
|
1004
|
+
fullText: string;
|
|
1005
|
+
operators?: NativeOperatorConfig;
|
|
1006
|
+
};
|
|
1007
|
+
type SharedNativeRedactTextOptions = SharedNativeRedactTextJsonOptions;
|
|
1008
|
+
type SharedNativeDiagnosticsJsonOptions = SharedNativeRedactTextJsonOptions;
|
|
1009
|
+
type SharedNativeDiagnosticsStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
1010
|
+
onBatch: NativeDiagnosticsBatchCallback;
|
|
1011
|
+
};
|
|
1012
|
+
type SharedNativeRedactTextStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
1013
|
+
onEvent: NativeResultEventCallback;
|
|
1014
|
+
};
|
|
1015
|
+
type NativeNormalizeOptions = {
|
|
1016
|
+
binding: NativeAnonymizeBinding;
|
|
1017
|
+
text: string;
|
|
1018
|
+
};
|
|
1019
|
+
type NativeAnonymizerFromConfigOptions = {
|
|
1020
|
+
binding: NativeAnonymizeBinding;
|
|
1021
|
+
config: NativePreparedSearchConfig;
|
|
1022
|
+
};
|
|
1023
|
+
type NativeAnonymizerFromPackageOptions = {
|
|
1024
|
+
binding: NativeAnonymizeBinding;
|
|
1025
|
+
packageBytes: Uint8Array;
|
|
1026
|
+
};
|
|
1027
|
+
type NativePipelineFromPackageOptions = NativeAnonymizerFromPackageOptions;
|
|
1028
|
+
type NativeBindingVersionOptions = {
|
|
1029
|
+
binding: NativeAnonymizeBinding;
|
|
1030
|
+
expectedVersion: string;
|
|
1031
|
+
};
|
|
1032
|
+
declare class PreparedNativeAnonymizer {
|
|
1033
|
+
#private;
|
|
1034
|
+
constructor(prepared: NativePreparedSearchBinding);
|
|
1035
|
+
prepareDiagnosticsJson(): string | null;
|
|
1036
|
+
prepare_diagnostics_json(): string | null;
|
|
1037
|
+
warmLazyRegex(): void;
|
|
1038
|
+
warm_lazy_regex(): void;
|
|
1039
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
1040
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
1041
|
+
redactStaticEntities(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1042
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1043
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1044
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1045
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1046
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1047
|
+
redactStaticEntitiesDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1048
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1049
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1050
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1051
|
+
redactStaticEntitiesSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1052
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1053
|
+
}
|
|
1054
|
+
declare class PreparedNativePipeline {
|
|
1055
|
+
#private;
|
|
1056
|
+
constructor(anonymizer: PreparedNativeAnonymizer);
|
|
1057
|
+
prepareDiagnosticsJson(): string | null;
|
|
1058
|
+
prepare_diagnostics_json(): string | null;
|
|
1059
|
+
warmLazyRegex(): void;
|
|
1060
|
+
warm_lazy_regex(): void;
|
|
1061
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
1062
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
1063
|
+
redactText(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1064
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1065
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1066
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1067
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1068
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1069
|
+
redactTextDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1070
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1071
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1072
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1073
|
+
redactTextSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1074
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1075
|
+
}
|
|
1076
|
+
declare const encodeNativeSearchConfig: (config: NativePreparedSearchConfig) => Uint8Array;
|
|
1077
|
+
declare const encodeNativeSearchConfigInput: (config: NativeSearchPackageInput) => Uint8Array;
|
|
1078
|
+
declare const getNativeBindingVersion: (binding: NativeAnonymizeBinding) => string;
|
|
1079
|
+
declare const native_package_version: (binding: NativeAnonymizeBinding) => string;
|
|
1080
|
+
declare const normalize_for_search: ({
|
|
1081
|
+
binding,
|
|
1082
|
+
text
|
|
1083
|
+
}: NativeNormalizeOptions) => string;
|
|
1084
|
+
declare const assertNativeBindingVersion: ({
|
|
1085
|
+
binding,
|
|
1086
|
+
expectedVersion
|
|
1087
|
+
}: NativeBindingVersionOptions) => void;
|
|
1088
|
+
declare const prepareNativeSearchPackage: ({
|
|
1089
|
+
binding,
|
|
1090
|
+
config,
|
|
1091
|
+
compressed
|
|
1092
|
+
}: NativeSearchPackageOptions) => Uint8Array;
|
|
1093
|
+
declare const prepare_search_package: ({
|
|
1094
|
+
binding,
|
|
1095
|
+
config,
|
|
1096
|
+
compressed
|
|
1097
|
+
}: SharedNativeSearchPackageOptions) => Uint8Array;
|
|
1098
|
+
declare const createNativeAnonymizerFromConfig: ({
|
|
1099
|
+
binding,
|
|
1100
|
+
config
|
|
1101
|
+
}: NativeAnonymizerFromConfigOptions) => PreparedNativeAnonymizer;
|
|
1102
|
+
declare const createNativeAnonymizerFromPackage: ({
|
|
1103
|
+
binding,
|
|
1104
|
+
packageBytes
|
|
1105
|
+
}: NativeAnonymizerFromPackageOptions) => PreparedNativeAnonymizer;
|
|
1106
|
+
declare const load_prepared_package: ({
|
|
1107
|
+
binding,
|
|
1108
|
+
packageBytes
|
|
1109
|
+
}: SharedNativePreparedPackageOptions) => PreparedNativeAnonymizer;
|
|
1110
|
+
declare const redact_text_json: ({
|
|
1111
|
+
binding,
|
|
1112
|
+
config,
|
|
1113
|
+
fullText,
|
|
1114
|
+
operators
|
|
1115
|
+
}: SharedNativeRedactTextJsonOptions) => string;
|
|
1116
|
+
declare const redact_text: ({
|
|
1117
|
+
binding,
|
|
1118
|
+
config,
|
|
1119
|
+
fullText,
|
|
1120
|
+
operators
|
|
1121
|
+
}: SharedNativeRedactTextOptions) => NativeStaticRedactionResult;
|
|
1122
|
+
declare const redact_text_stream_json: ({
|
|
1123
|
+
binding,
|
|
1124
|
+
config,
|
|
1125
|
+
fullText,
|
|
1126
|
+
operators,
|
|
1127
|
+
onEvent
|
|
1128
|
+
}: SharedNativeRedactTextStreamJsonOptions) => string | null;
|
|
1129
|
+
declare const diagnostics_json: ({
|
|
1130
|
+
binding,
|
|
1131
|
+
config,
|
|
1132
|
+
fullText,
|
|
1133
|
+
operators
|
|
1134
|
+
}: SharedNativeDiagnosticsJsonOptions) => string | null;
|
|
1135
|
+
declare const diagnostics_stream_json: ({
|
|
1136
|
+
binding,
|
|
1137
|
+
config,
|
|
1138
|
+
fullText,
|
|
1139
|
+
operators,
|
|
1140
|
+
onBatch
|
|
1141
|
+
}: SharedNativeDiagnosticsStreamJsonOptions) => string | null;
|
|
1142
|
+
declare const summary_diagnostics_json: ({
|
|
1143
|
+
binding,
|
|
1144
|
+
config,
|
|
1145
|
+
fullText,
|
|
1146
|
+
operators
|
|
1147
|
+
}: SharedNativeDiagnosticsJsonOptions) => string | null;
|
|
1148
|
+
declare const createNativePipelineFromPackage: ({
|
|
1149
|
+
binding,
|
|
1150
|
+
packageBytes
|
|
1151
|
+
}: NativePipelineFromPackageOptions) => PreparedNativePipeline;
|
|
1152
|
+
declare const PreparedSearch: typeof PreparedNativeAnonymizer;
|
|
1153
|
+
type PreparedSearch = PreparedNativeAnonymizer;
|
|
1154
|
+
declare const PreparedAnonymizer: typeof PreparedNativeAnonymizer;
|
|
1155
|
+
type PreparedAnonymizer = PreparedNativeAnonymizer;
|
|
1156
|
+
//#endregion
|
|
1157
|
+
export { Entity as $, createNativePipelineFromPackage as A, prepare_search_package as B, SharedNativeRedactTextJsonOptions as C, assertNativeBindingVersion as D, SharedNativeSearchPackageOptions as E, getNativeBindingVersion as F, NativePreparedSearchConfig as G, redact_text_json as H, load_prepared_package as I, CustomDenyListEntry as J, PipelineContext as K, native_package_version as L, diagnostics_stream_json as M, encodeNativeSearchConfig as N, createNativeAnonymizerFromConfig as O, encodeNativeSearchConfigInput as P, DictionaryMeta as Q, normalize_for_search as R, SharedNativePreparedPackageOptions as S, SharedNativeRedactTextStreamJsonOptions as T, redact_text_stream_json as U, redact_text as V, summary_diagnostics_json as W, DenyListCategory as X, CustomRegexPattern as Y, Dictionaries as Z, PreparedNativeAnonymizer as _, NativeDiagnosticsBatchCallback as a, ReviewedEntity as at, SharedNativeDiagnosticsJsonOptions as b, NativePipelineEntity as c, TriggerRule as ct, NativeRedactionResult as d, GazetteerEntry as et, NativeResultEventCallback as f, PreparedAnonymizer as g, NativeStaticRedactionResult as h, NativeBindingVersionOptions as i, ReviewDecision as it, diagnostics_json as j, createNativeAnonymizerFromPackage as k, NativePipelineFromPackageOptions as l, TriggerStrategy as lt, NativeSearchPackageOptions as m, NativeAnonymizerFromConfigOptions as n, PipelineConfig as nt, NativeNormalizeOptions as o, TriggerExtension as ot, NativeSearchPackageInput as p, AnonymisationOperator as q, NativeAnonymizerFromPackageOptions as r, RedactionResult as rt, NativeOperatorConfig as s, TriggerGroupConfig as st, NativeAnonymizeBinding as t, OperatorConfig as tt, NativePreparedSearchBinding as u, TriggerValidation as ut, PreparedNativePipeline as v, SharedNativeRedactTextOptions as w, SharedNativeDiagnosticsStreamJsonOptions as x, PreparedSearch as y, prepareNativeSearchPackage as z };
|
|
1158
|
+
//# sourceMappingURL=native.d.mts.map
|