@stll/anonymize-wasm 1.5.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -2
- package/README.md +93 -4
- package/dist/native/index.wasi-browser.js +12002 -0
- package/dist/native/index.wasi.cjs +122 -0
- package/dist/native/index.wasm32-wasi.wasm +0 -0
- package/dist/native/native-pipeline.cs.stlanonpkg +0 -0
- package/dist/native/native-pipeline.de.stlanonpkg +0 -0
- package/dist/native/native-pipeline.en.stlanonpkg +0 -0
- package/dist/native/native-pipeline.stlanonpkg +0 -0
- package/dist/native/wasi-worker-browser.mjs +10766 -0
- package/dist/native/wasi-worker.mjs +63 -0
- package/dist/vite.d.mts +18 -2
- package/dist/vite.mjs +89 -12
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +553 -1102
- package/dist/wasm.mjs +359 -16223
- package/dist/wasm.mjs.map +1 -1
- package/package.json +15 -9
- package/dist/address-boundaries.mjs +0 -195
- package/dist/address-boundaries.mjs.map +0 -1
- package/dist/address-prepositions.mjs +0 -182
- package/dist/address-prepositions.mjs.map +0 -1
- package/dist/address-stop-keywords.mjs +0 -137
- package/dist/address-stop-keywords.mjs.map +0 -1
- package/dist/address-stopwords.mjs +0 -84
- package/dist/address-stopwords.mjs.map +0 -1
- package/dist/allow-list.mjs +0 -196
- package/dist/allow-list.mjs.map +0 -1
- package/dist/clause-noun-heads.mjs +0 -75
- package/dist/clause-noun-heads.mjs.map +0 -1
- package/dist/common-words-en.mjs +0 -9887
- package/dist/common-words-en.mjs.map +0 -1
- package/dist/coreference.cs.mjs +0 -14
- package/dist/coreference.cs.mjs.map +0 -1
- package/dist/coreference.de.mjs +0 -14
- package/dist/coreference.de.mjs.map +0 -1
- package/dist/coreference.en.mjs +0 -14
- package/dist/coreference.en.mjs.map +0 -1
- package/dist/coreference.es.mjs +0 -22
- package/dist/coreference.es.mjs.map +0 -1
- package/dist/coreference.fr.mjs +0 -32
- package/dist/coreference.fr.mjs.map +0 -1
- package/dist/coreference.it.mjs +0 -27
- package/dist/coreference.it.mjs.map +0 -1
- package/dist/coreference.pl.mjs +0 -27
- package/dist/coreference.pl.mjs.map +0 -1
- package/dist/coreference.pt-br.mjs +0 -14
- package/dist/coreference.pt-br.mjs.map +0 -1
- package/dist/coreference.sk.mjs +0 -27
- package/dist/coreference.sk.mjs.map +0 -1
- package/dist/currencies.mjs +0 -231
- package/dist/currencies.mjs.map +0 -1
- package/dist/date-months.mjs +0 -618
- package/dist/date-months.mjs.map +0 -1
- package/dist/document-structure-headings.mjs +0 -90
- package/dist/document-structure-headings.mjs.map +0 -1
- package/dist/generic-roles.mjs +0 -244
- package/dist/generic-roles.mjs.map +0 -1
- package/dist/hotword-rules.mjs +0 -149
- package/dist/hotword-rules.mjs.map +0 -1
- package/dist/legal-form-leading-clauses.mjs +0 -23
- package/dist/legal-form-leading-clauses.mjs.map +0 -1
- package/dist/legal-forms.mjs +0 -2115
- package/dist/legal-forms.mjs.map +0 -1
- package/dist/legal-role-heads.cs.mjs +0 -42
- package/dist/legal-role-heads.cs.mjs.map +0 -1
- package/dist/legal-role-heads.de.mjs +0 -33
- package/dist/legal-role-heads.de.mjs.map +0 -1
- package/dist/legal-role-heads.en.mjs +0 -37
- package/dist/legal-role-heads.en.mjs.map +0 -1
- package/dist/legal-role-heads.es.mjs +0 -54
- package/dist/legal-role-heads.es.mjs.map +0 -1
- package/dist/legal-role-heads.fr.mjs +0 -72
- package/dist/legal-role-heads.fr.mjs.map +0 -1
- package/dist/legal-role-heads.it.mjs +0 -68
- package/dist/legal-role-heads.it.mjs.map +0 -1
- package/dist/legal-role-heads.pl.mjs +0 -84
- package/dist/legal-role-heads.pl.mjs.map +0 -1
- package/dist/legal-role-heads.pt-br.mjs +0 -63
- package/dist/legal-role-heads.pt-br.mjs.map +0 -1
- package/dist/legal-role-heads.sk.mjs +0 -80
- package/dist/legal-role-heads.sk.mjs.map +0 -1
- package/dist/manifest.mjs +0 -69
- package/dist/manifest.mjs.map +0 -1
- package/dist/names-exclusions.mjs +0 -223
- package/dist/names-exclusions.mjs.map +0 -1
- package/dist/names-first.mjs +0 -418
- package/dist/names-first.mjs.map +0 -1
- package/dist/names-nw-ar.mjs +0 -202
- package/dist/names-nw-ar.mjs.map +0 -1
- package/dist/names-nw-excluded-allcaps.mjs +0 -112
- package/dist/names-nw-excluded-allcaps.mjs.map +0 -1
- package/dist/names-nw-fil.mjs +0 -202
- package/dist/names-nw-fil.mjs.map +0 -1
- package/dist/names-nw-id.mjs +0 -210
- package/dist/names-nw-id.mjs.map +0 -1
- package/dist/names-nw-in.mjs +0 -526
- package/dist/names-nw-in.mjs.map +0 -1
- package/dist/names-nw-ja-latn.mjs +0 -260
- package/dist/names-nw-ja-latn.mjs.map +0 -1
- package/dist/names-nw-ko.mjs +0 -162
- package/dist/names-nw-ko.mjs.map +0 -1
- package/dist/names-nw-th.mjs +0 -188
- package/dist/names-nw-th.mjs.map +0 -1
- package/dist/names-nw-vi.mjs +0 -151
- package/dist/names-nw-vi.mjs.map +0 -1
- package/dist/names-nw-zh-latn.mjs +0 -197
- package/dist/names-nw-zh-latn.mjs.map +0 -1
- package/dist/names-surnames.mjs +0 -113
- package/dist/names-surnames.mjs.map +0 -1
- package/dist/names-title-tokens.mjs +0 -40
- package/dist/names-title-tokens.mjs.map +0 -1
- package/dist/person-stopwords.mjs +0 -205
- package/dist/person-stopwords.mjs.map +0 -1
- package/dist/section-headings.mjs +0 -64
- package/dist/section-headings.mjs.map +0 -1
- package/dist/sentence-verb-indicators.mjs +0 -232
- package/dist/sentence-verb-indicators.mjs.map +0 -1
- package/dist/signing-clauses.mjs +0 -78
- package/dist/signing-clauses.mjs.map +0 -1
- package/dist/stopwords.mjs +0 -9915
- package/dist/stopwords.mjs.map +0 -1
- package/dist/structural-single-cap-prefixes.mjs +0 -99
- package/dist/structural-single-cap-prefixes.mjs.map +0 -1
- package/dist/triggers.cs.mjs +0 -569
- package/dist/triggers.cs.mjs.map +0 -1
- package/dist/triggers.de.mjs +0 -139
- package/dist/triggers.de.mjs.map +0 -1
- package/dist/triggers.en.mjs +0 -119
- package/dist/triggers.en.mjs.map +0 -1
- package/dist/triggers.es.mjs +0 -96
- package/dist/triggers.es.mjs.map +0 -1
- package/dist/triggers.fr.mjs +0 -275
- package/dist/triggers.fr.mjs.map +0 -1
- package/dist/triggers.global.mjs +0 -79
- package/dist/triggers.global.mjs.map +0 -1
- package/dist/triggers.hu.mjs +0 -41
- package/dist/triggers.hu.mjs.map +0 -1
- package/dist/triggers.it.mjs +0 -74
- package/dist/triggers.it.mjs.map +0 -1
- package/dist/triggers.pl.mjs +0 -271
- package/dist/triggers.pl.mjs.map +0 -1
- package/dist/triggers.pt-br.mjs +0 -193
- package/dist/triggers.pt-br.mjs.map +0 -1
- package/dist/triggers.ro.mjs +0 -59
- package/dist/triggers.ro.mjs.map +0 -1
- package/dist/triggers.sk.mjs +0 -555
- package/dist/triggers.sk.mjs.map +0 -1
- package/dist/triggers.sv.mjs +0 -58
- package/dist/triggers.sv.mjs.map +0 -1
- package/dist/year-words.mjs +0 -62
- package/dist/year-words.mjs.map +0 -1
package/dist/wasm.d.mts
CHANGED
|
@@ -1,8 +1,316 @@
|
|
|
1
1
|
import { a as OPERATOR_TYPES, i as DetectionSource, n as DETECTION_SOURCES, o as OperatorType, r as DETECTOR_PRIORITY, t as DEFAULT_ENTITY_LABELS } from "./constants2.mjs";
|
|
2
|
-
import { Validator } from "@stll/stdnum";
|
|
3
|
-
import { Match, PatternEntry, TextSearch } from "@stll/text-search";
|
|
4
|
-
import { Tokenizer } from "@huggingface/tokenizers";
|
|
5
2
|
|
|
3
|
+
//#region src/native-search-config.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* Structural type for the prepared static-search config the native binding
|
|
6
|
+
* consumes and the Rust assembler emits (`assembleStaticSearchConfigJson`).
|
|
7
|
+
*
|
|
8
|
+
* This config used to be built in TypeScript by `build-unified-search.ts`; that
|
|
9
|
+
* layer was retired in favor of the Rust assembler
|
|
10
|
+
* (`crates/anonymize-adapter-contract` `assemble_static_search_config`). The
|
|
11
|
+
* type now lives here as a pure, dependency-free description of the JSON the
|
|
12
|
+
* binding accepts on its `fromConfigJsonBytes` / prepare paths, so callers that
|
|
13
|
+
* hold a pre-assembled config keep a precise type without pulling in the
|
|
14
|
+
* deleted detector modules.
|
|
15
|
+
*/
|
|
16
|
+
type PatternSlice = {
|
|
17
|
+
start: number;
|
|
18
|
+
end: number;
|
|
19
|
+
};
|
|
20
|
+
type NativeSearchPatternKind = "literal" | "literal-with-options" | "regex" | "fuzzy";
|
|
21
|
+
type NativeSearchPattern = {
|
|
22
|
+
kind: NativeSearchPatternKind;
|
|
23
|
+
pattern: string;
|
|
24
|
+
distance?: number;
|
|
25
|
+
case_insensitive?: boolean;
|
|
26
|
+
whole_words?: boolean;
|
|
27
|
+
lazy?: boolean;
|
|
28
|
+
prefilter_any?: string[];
|
|
29
|
+
prefilter_case_insensitive?: boolean;
|
|
30
|
+
prefilter_regex?: string;
|
|
31
|
+
prefilter_window_bytes?: number;
|
|
32
|
+
prepared_artifact_policy?: "include" | "omit";
|
|
33
|
+
};
|
|
34
|
+
type NativeSearchOptions = {
|
|
35
|
+
literal_case_insensitive?: boolean;
|
|
36
|
+
literal_whole_words?: boolean;
|
|
37
|
+
regex_whole_words?: boolean;
|
|
38
|
+
regex_overlap_all?: boolean;
|
|
39
|
+
regex_artifact_policy?: "include" | "omit";
|
|
40
|
+
fuzzy_case_insensitive?: boolean;
|
|
41
|
+
fuzzy_whole_words?: boolean;
|
|
42
|
+
fuzzy_normalize_diacritics?: boolean;
|
|
43
|
+
};
|
|
44
|
+
type NativeRegexMatchMeta = {
|
|
45
|
+
label: string;
|
|
46
|
+
score: number;
|
|
47
|
+
source_detail?: string;
|
|
48
|
+
requires_validation?: boolean;
|
|
49
|
+
validator_id?: string;
|
|
50
|
+
validator_input?: string;
|
|
51
|
+
min_byte_length?: number;
|
|
52
|
+
};
|
|
53
|
+
type NativeSigningPlaceGuardData = {
|
|
54
|
+
prefix_phrases: string[];
|
|
55
|
+
suffix_phrases: string[];
|
|
56
|
+
};
|
|
57
|
+
type NativeDenyListFilterData = {
|
|
58
|
+
stopwords: string[];
|
|
59
|
+
allow_list: string[];
|
|
60
|
+
person_stopwords: string[];
|
|
61
|
+
person_trailing_nouns: string[];
|
|
62
|
+
address_stopwords: string[];
|
|
63
|
+
address_jurisdiction_prefixes: string[];
|
|
64
|
+
street_types: string[];
|
|
65
|
+
address_component_terms: string[];
|
|
66
|
+
ambiguous_street_type_terms: string[];
|
|
67
|
+
first_names: string[];
|
|
68
|
+
generic_roles: string[];
|
|
69
|
+
number_abbrev_prefixes: string[];
|
|
70
|
+
sentence_starters: string[];
|
|
71
|
+
trailing_address_word_exclusions: string[];
|
|
72
|
+
document_heading_words: string[];
|
|
73
|
+
document_heading_ordinal_markers: string[];
|
|
74
|
+
defined_term_cues: string[];
|
|
75
|
+
signing_place_guards: NativeSigningPlaceGuardData[];
|
|
76
|
+
};
|
|
77
|
+
type NativeDenyListMatchData = {
|
|
78
|
+
labels?: string[][];
|
|
79
|
+
label_table?: string[];
|
|
80
|
+
label_indices?: number[][];
|
|
81
|
+
custom_labels?: string[][];
|
|
82
|
+
custom_label_indices?: number[][];
|
|
83
|
+
originals: string[];
|
|
84
|
+
sources?: string[][];
|
|
85
|
+
source_table?: string[];
|
|
86
|
+
source_indices?: number[][];
|
|
87
|
+
filters?: NativeDenyListFilterData;
|
|
88
|
+
};
|
|
89
|
+
type NativeTriggerStrategy = {
|
|
90
|
+
type: "to-next-comma";
|
|
91
|
+
stop_words?: string[];
|
|
92
|
+
max_length?: number;
|
|
93
|
+
} | {
|
|
94
|
+
type: "to-end-of-line";
|
|
95
|
+
} | {
|
|
96
|
+
type: "n-words";
|
|
97
|
+
count: number;
|
|
98
|
+
} | {
|
|
99
|
+
type: "company-id-value";
|
|
100
|
+
} | {
|
|
101
|
+
type: "address";
|
|
102
|
+
max_chars?: number;
|
|
103
|
+
} | {
|
|
104
|
+
type: "match-pattern";
|
|
105
|
+
pattern: string;
|
|
106
|
+
flags?: string;
|
|
107
|
+
};
|
|
108
|
+
type NativeTriggerValidation = {
|
|
109
|
+
type: "starts-uppercase";
|
|
110
|
+
} | {
|
|
111
|
+
type: "min-length";
|
|
112
|
+
min: number;
|
|
113
|
+
} | {
|
|
114
|
+
type: "max-length";
|
|
115
|
+
max: number;
|
|
116
|
+
} | {
|
|
117
|
+
type: "no-digits";
|
|
118
|
+
} | {
|
|
119
|
+
type: "has-digits";
|
|
120
|
+
} | {
|
|
121
|
+
type: "matches-pattern";
|
|
122
|
+
pattern: string;
|
|
123
|
+
flags?: string;
|
|
124
|
+
} | {
|
|
125
|
+
type: "valid-id";
|
|
126
|
+
validator: string;
|
|
127
|
+
};
|
|
128
|
+
type NativeTriggerRule = {
|
|
129
|
+
trigger: string;
|
|
130
|
+
label: string;
|
|
131
|
+
strategy: NativeTriggerStrategy;
|
|
132
|
+
validations: NativeTriggerValidation[];
|
|
133
|
+
include_trigger: boolean;
|
|
134
|
+
};
|
|
135
|
+
type NativeTriggerData = {
|
|
136
|
+
rules: NativeTriggerRule[];
|
|
137
|
+
address_stop_keywords: string[];
|
|
138
|
+
party_position_terms: string[];
|
|
139
|
+
post_nominals: string[];
|
|
140
|
+
sentence_terminal_currency_terms: string[];
|
|
141
|
+
phone_extension_labels: string[];
|
|
142
|
+
number_markers: string[];
|
|
143
|
+
number_labels: string[];
|
|
144
|
+
};
|
|
145
|
+
type NativeLegalFormData = {
|
|
146
|
+
suffixes: string[];
|
|
147
|
+
normalized_boundary_suffixes: string[];
|
|
148
|
+
normalized_in_name_words: string[];
|
|
149
|
+
normalized_suffix_words: string[];
|
|
150
|
+
role_heads: string[];
|
|
151
|
+
sentence_verb_indicators: string[];
|
|
152
|
+
clause_noun_heads: string[];
|
|
153
|
+
connector_prose_heads: string[];
|
|
154
|
+
structural_single_cap_prefixes: string[];
|
|
155
|
+
leading_clause_phrases: string[];
|
|
156
|
+
leading_clause_direct_prefixes: string[];
|
|
157
|
+
connector_words: string[];
|
|
158
|
+
and_connector_words: string[];
|
|
159
|
+
in_name_prepositions: string[];
|
|
160
|
+
company_suffix_words: string[];
|
|
161
|
+
comma_gated_direct_prefixes: string[];
|
|
162
|
+
};
|
|
163
|
+
type NativeDateMonthData = Record<string, string[]>;
|
|
164
|
+
type NativeYearWordData = Record<string, string[]>;
|
|
165
|
+
type NativeDateData = {
|
|
166
|
+
month_names_by_language: NativeDateMonthData;
|
|
167
|
+
year_words_by_language: NativeYearWordData;
|
|
168
|
+
};
|
|
169
|
+
type NativeMonetaryData = {
|
|
170
|
+
currencies: {
|
|
171
|
+
codes: string[];
|
|
172
|
+
symbols: string[];
|
|
173
|
+
local_names: string[];
|
|
174
|
+
};
|
|
175
|
+
amount_words: {
|
|
176
|
+
written_amount_patterns: Array<{
|
|
177
|
+
keywords: string[];
|
|
178
|
+
}>;
|
|
179
|
+
magnitude_suffixes: Array<{
|
|
180
|
+
words: string[];
|
|
181
|
+
abbreviations_case_insensitive: string[];
|
|
182
|
+
abbreviations_case_sensitive: string[];
|
|
183
|
+
}>;
|
|
184
|
+
share_quantity_terms: Array<{
|
|
185
|
+
modifiers: string[];
|
|
186
|
+
nouns: string[];
|
|
187
|
+
}>;
|
|
188
|
+
};
|
|
189
|
+
};
|
|
190
|
+
type NativeAddressSeedData = {
|
|
191
|
+
boundary_words: string[];
|
|
192
|
+
br_cep_cue_words: string[];
|
|
193
|
+
unit_abbreviations: string[];
|
|
194
|
+
};
|
|
195
|
+
type NativeAddressContextData = {
|
|
196
|
+
address_prepositions: string[];
|
|
197
|
+
temporal_prepositions: string[];
|
|
198
|
+
street_abbreviations: string[];
|
|
199
|
+
bare_house_stopwords: string[];
|
|
200
|
+
};
|
|
201
|
+
type NativeCoreferencePatternData = {
|
|
202
|
+
pattern: string;
|
|
203
|
+
flags: string;
|
|
204
|
+
};
|
|
205
|
+
type NativeCoreferenceData = {
|
|
206
|
+
definition_patterns: NativeCoreferencePatternData[];
|
|
207
|
+
role_stop_terms: string[];
|
|
208
|
+
legal_form_aliases: string[];
|
|
209
|
+
organization_suffixes: string[];
|
|
210
|
+
organization_determiners: string[];
|
|
211
|
+
};
|
|
212
|
+
type NativeNameCorpusData = {
|
|
213
|
+
first_names: string[];
|
|
214
|
+
surnames: string[];
|
|
215
|
+
title_tokens: string[];
|
|
216
|
+
title_abbreviations: string[];
|
|
217
|
+
excluded_words: string[];
|
|
218
|
+
common_words: string[];
|
|
219
|
+
non_western_names: string[];
|
|
220
|
+
excluded_all_caps: string[];
|
|
221
|
+
ja_suffixes: string[];
|
|
222
|
+
arabic_connectors: string[];
|
|
223
|
+
relation_connectors: string[];
|
|
224
|
+
hyphenated_prefixes: string[];
|
|
225
|
+
cjk_non_person_terms: string[];
|
|
226
|
+
cjk_surname_starters: string[];
|
|
227
|
+
organization_terms: string[];
|
|
228
|
+
};
|
|
229
|
+
type NativeNameCorpusMode = "full" | "supplemental";
|
|
230
|
+
type NativeZonePatternData = {
|
|
231
|
+
pattern: string;
|
|
232
|
+
flags: string;
|
|
233
|
+
};
|
|
234
|
+
type NativeZoneSigningClauseData = {
|
|
235
|
+
prefix: string;
|
|
236
|
+
suffix: string;
|
|
237
|
+
prepositions: string[];
|
|
238
|
+
};
|
|
239
|
+
type NativeZoneData = {
|
|
240
|
+
section_heading_patterns: NativeZonePatternData[];
|
|
241
|
+
signing_clauses: NativeZoneSigningClauseData[];
|
|
242
|
+
};
|
|
243
|
+
type NativeCountryData = {
|
|
244
|
+
labels: string[];
|
|
245
|
+
isoCodes: string[];
|
|
246
|
+
variants: Array<"name" | "alias" | "alpha3" | "alpha2">;
|
|
247
|
+
};
|
|
248
|
+
type NativeGazetteerData = {
|
|
249
|
+
labels: string[];
|
|
250
|
+
is_fuzzy: boolean[];
|
|
251
|
+
};
|
|
252
|
+
type NativeHotwordRule = {
|
|
253
|
+
hotwords: string[];
|
|
254
|
+
target_labels: string[];
|
|
255
|
+
score_adjustment: number;
|
|
256
|
+
reclassify_to?: string;
|
|
257
|
+
proximity_before: number;
|
|
258
|
+
proximity_after: number;
|
|
259
|
+
};
|
|
260
|
+
type NativeHotwordRuleData = {
|
|
261
|
+
rules: NativeHotwordRule[];
|
|
262
|
+
pattern_rule_indices: number[];
|
|
263
|
+
};
|
|
264
|
+
type NativeSignatureData = {
|
|
265
|
+
labels: string[];
|
|
266
|
+
witness_phrases: string[];
|
|
267
|
+
name_particles: string[];
|
|
268
|
+
post_nominal_suffixes: string[];
|
|
269
|
+
organization_suffixes: string[];
|
|
270
|
+
image_stub_prefixes: string[];
|
|
271
|
+
};
|
|
272
|
+
type NativePreparedSearchConfig = {
|
|
273
|
+
regex_patterns: NativeSearchPattern[];
|
|
274
|
+
custom_regex_patterns: NativeSearchPattern[];
|
|
275
|
+
literal_patterns: NativeSearchPattern[];
|
|
276
|
+
regex_options: NativeSearchOptions;
|
|
277
|
+
custom_regex_options: NativeSearchOptions;
|
|
278
|
+
literal_options: NativeSearchOptions;
|
|
279
|
+
literal_patterns_from_deny_list_data?: boolean;
|
|
280
|
+
allowed_labels: string[];
|
|
281
|
+
threshold: number;
|
|
282
|
+
confidence_boost: boolean;
|
|
283
|
+
slices: {
|
|
284
|
+
regex: PatternSlice;
|
|
285
|
+
custom_regex: PatternSlice;
|
|
286
|
+
legal_forms?: PatternSlice;
|
|
287
|
+
triggers?: PatternSlice;
|
|
288
|
+
deny_list: PatternSlice;
|
|
289
|
+
street_types?: PatternSlice;
|
|
290
|
+
gazetteer: PatternSlice;
|
|
291
|
+
countries: PatternSlice;
|
|
292
|
+
hotwords?: PatternSlice;
|
|
293
|
+
};
|
|
294
|
+
regex_meta: NativeRegexMatchMeta[];
|
|
295
|
+
custom_regex_meta: NativeRegexMatchMeta[];
|
|
296
|
+
deny_list_data?: NativeDenyListMatchData;
|
|
297
|
+
false_positive_filters?: NativeDenyListFilterData;
|
|
298
|
+
gazetteer_data?: NativeGazetteerData;
|
|
299
|
+
country_data?: NativeCountryData;
|
|
300
|
+
hotword_data?: NativeHotwordRuleData;
|
|
301
|
+
trigger_data?: NativeTriggerData;
|
|
302
|
+
legal_form_data?: NativeLegalFormData;
|
|
303
|
+
address_seed_data?: NativeAddressSeedData;
|
|
304
|
+
zone_data?: NativeZoneData;
|
|
305
|
+
address_context_data?: NativeAddressContextData;
|
|
306
|
+
coreference_data?: NativeCoreferenceData;
|
|
307
|
+
name_corpus_data?: NativeNameCorpusData;
|
|
308
|
+
signature_data?: NativeSignatureData;
|
|
309
|
+
name_corpus_mode?: NativeNameCorpusMode;
|
|
310
|
+
date_data?: NativeDateData;
|
|
311
|
+
monetary_data?: NativeMonetaryData;
|
|
312
|
+
};
|
|
313
|
+
//#endregion
|
|
6
314
|
//#region src/types.d.ts
|
|
7
315
|
/**
|
|
8
316
|
* Fields shared by every entity span in the source text.
|
|
@@ -53,146 +361,6 @@ type ReviewedEntity = Entity & {
|
|
|
53
361
|
decision?: ReviewDecision;
|
|
54
362
|
originalLabel?: string;
|
|
55
363
|
};
|
|
56
|
-
/**
|
|
57
|
-
* A single entry in the workspace-scoped gazetteer
|
|
58
|
-
* (deny list). Persisted in IndexedDB.
|
|
59
|
-
*/
|
|
60
|
-
type GazetteerEntry = {
|
|
61
|
-
id: string;
|
|
62
|
-
canonical: string;
|
|
63
|
-
label: string;
|
|
64
|
-
variants: string[];
|
|
65
|
-
workspaceId: string;
|
|
66
|
-
createdAt: number;
|
|
67
|
-
source: "manual" | "confirmed-from-model";
|
|
68
|
-
};
|
|
69
|
-
/** Extraction strategy — closed discriminated union. */
|
|
70
|
-
type TriggerStrategy = {
|
|
71
|
-
type: "to-next-comma";
|
|
72
|
-
/**
|
|
73
|
-
* Optional list of lowercase keywords that terminate
|
|
74
|
-
* the value scan, in addition to commas/newlines. Useful
|
|
75
|
-
* for triggers like court names that may continue past
|
|
76
|
-
* a missing comma into adjacent clause text ("Městským
|
|
77
|
-
* soudem v Praze dne 1. 1. 2020"); listing `"dne"` here
|
|
78
|
-
* stops the scan at the date boundary. Matched on a
|
|
79
|
-
* word-boundary, case-insensitive.
|
|
80
|
-
*/
|
|
81
|
-
stopWords?: string[];
|
|
82
|
-
/**
|
|
83
|
-
* Hard cap on the captured span length, in characters,
|
|
84
|
-
* regardless of where the next comma / stop char sits.
|
|
85
|
-
* Use for triggers that label short formulaic phrases
|
|
86
|
-
* ("State of Delaware") and must not absorb the rest
|
|
87
|
-
* of a long forum-selection clause when the comma is
|
|
88
|
-
* sentences away. Falls back to the default 100-char
|
|
89
|
-
* fallback when omitted.
|
|
90
|
-
*/
|
|
91
|
-
maxLength?: number;
|
|
92
|
-
} | {
|
|
93
|
-
type: "to-end-of-line";
|
|
94
|
-
} | {
|
|
95
|
-
type: "n-words";
|
|
96
|
-
count: number;
|
|
97
|
-
} | {
|
|
98
|
-
type: "company-id-value";
|
|
99
|
-
} | {
|
|
100
|
-
type: "address";
|
|
101
|
-
maxChars?: number;
|
|
102
|
-
} | {
|
|
103
|
-
/**
|
|
104
|
-
* Extract the first regex match in the value text.
|
|
105
|
-
* Useful for shape-bounded values that follow a
|
|
106
|
-
* label on the same line as other fields, where
|
|
107
|
-
* `to-end-of-line` would over-capture. The pattern
|
|
108
|
-
* is anchored to the start of the (already
|
|
109
|
-
* leading-whitespace-stripped) value, so use
|
|
110
|
-
* `(?:.*?)` prefix only when intentional.
|
|
111
|
-
*/
|
|
112
|
-
type: "match-pattern";
|
|
113
|
-
pattern: string;
|
|
114
|
-
flags?: string;
|
|
115
|
-
};
|
|
116
|
-
/** Validation rules — closed discriminated union. */
|
|
117
|
-
type TriggerValidation = {
|
|
118
|
-
type: "starts-uppercase";
|
|
119
|
-
} | {
|
|
120
|
-
type: "min-length";
|
|
121
|
-
min: number;
|
|
122
|
-
} | {
|
|
123
|
-
type: "max-length";
|
|
124
|
-
max: number;
|
|
125
|
-
} | {
|
|
126
|
-
type: "no-digits";
|
|
127
|
-
} | {
|
|
128
|
-
type: "has-digits";
|
|
129
|
-
} | {
|
|
130
|
-
type: "matches-pattern";
|
|
131
|
-
pattern: string;
|
|
132
|
-
flags?: string;
|
|
133
|
-
}
|
|
134
|
-
/**
|
|
135
|
-
* Run a named stdnum validator (checksum + length)
|
|
136
|
-
* against the captured value. Keeps the trigger
|
|
137
|
-
* path symmetrical with the formatted-regex
|
|
138
|
-
* detectors so e.g. `CPF nº 00000000000` does not
|
|
139
|
-
* survive as a tax-ID entity.
|
|
140
|
-
*/
|
|
141
|
-
| {
|
|
142
|
-
type: "valid-id";
|
|
143
|
-
validator: ValidIdValidator;
|
|
144
|
-
};
|
|
145
|
-
/** Built-in stdnum validators that can be referenced
|
|
146
|
-
* by `valid-id` validations. */
|
|
147
|
-
type ValidIdValidator = "br.cpf" | "br.cnpj" | "us.rtn";
|
|
148
|
-
/** Auto-generated trigger variants — closed set. */
|
|
149
|
-
type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
|
|
150
|
-
/** V2 trigger config entry (JSON shape). */
|
|
151
|
-
type TriggerGroupConfig = {
|
|
152
|
-
id?: string;
|
|
153
|
-
triggers: string[];
|
|
154
|
-
label: string;
|
|
155
|
-
strategy: TriggerStrategy;
|
|
156
|
-
extensions?: TriggerExtension[];
|
|
157
|
-
validations?: TriggerValidation[];
|
|
158
|
-
/** When true, include the trigger text in the
|
|
159
|
-
* entity span (e.g., court names). */
|
|
160
|
-
includeTrigger?: boolean;
|
|
161
|
-
};
|
|
162
|
-
/** Compiled validation with pre-built regex. */
|
|
163
|
-
type CompiledValidation = {
|
|
164
|
-
type: "starts-uppercase";
|
|
165
|
-
re: RegExp;
|
|
166
|
-
} | {
|
|
167
|
-
type: "min-length";
|
|
168
|
-
min: number;
|
|
169
|
-
} | {
|
|
170
|
-
type: "max-length";
|
|
171
|
-
max: number;
|
|
172
|
-
} | {
|
|
173
|
-
type: "no-digits";
|
|
174
|
-
re: RegExp;
|
|
175
|
-
} | {
|
|
176
|
-
type: "has-digits";
|
|
177
|
-
re: RegExp;
|
|
178
|
-
} | {
|
|
179
|
-
type: "matches-pattern";
|
|
180
|
-
re: RegExp;
|
|
181
|
-
} | {
|
|
182
|
-
type: "valid-id";
|
|
183
|
-
check: (value: string) => boolean;
|
|
184
|
-
};
|
|
185
|
-
/**
|
|
186
|
-
* Runtime rule — one per trigger string after
|
|
187
|
-
* expansion. Fed to the Aho-Corasick automaton.
|
|
188
|
-
*/
|
|
189
|
-
type TriggerRule = {
|
|
190
|
-
trigger: string;
|
|
191
|
-
label: string;
|
|
192
|
-
strategy: TriggerStrategy;
|
|
193
|
-
validations: CompiledValidation[];
|
|
194
|
-
includeTrigger: boolean;
|
|
195
|
-
};
|
|
196
364
|
/** Per-label operator selection. Key is the entity label. */
|
|
197
365
|
type OperatorConfig = {
|
|
198
366
|
/** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
|
|
@@ -222,507 +390,213 @@ type RedactionResult = {
|
|
|
222
390
|
operatorMap: Map<string, OperatorType>;
|
|
223
391
|
entityCount: number;
|
|
224
392
|
};
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
*/
|
|
245
|
-
type CustomDenyListEntry = {
|
|
246
|
-
value: string;
|
|
393
|
+
//#endregion
|
|
394
|
+
//#region src/native.d.ts
|
|
395
|
+
type NativeBindingOperatorConfig = {
|
|
396
|
+
operators?: Record<string, OperatorType>;
|
|
397
|
+
redactString?: string;
|
|
398
|
+
};
|
|
399
|
+
type NativeDiagnosticsBatchCallback = (diagnosticsJson: string) => void;
|
|
400
|
+
type NativeResultEventCallback = (eventJson: string) => void;
|
|
401
|
+
type NativeBindingRedactionEntry = {
|
|
402
|
+
placeholder: string;
|
|
403
|
+
original: string;
|
|
404
|
+
};
|
|
405
|
+
type NativeBindingOperatorEntry = {
|
|
406
|
+
placeholder: string;
|
|
407
|
+
operator: OperatorType;
|
|
408
|
+
};
|
|
409
|
+
type NativeBindingPipelineEntity = {
|
|
410
|
+
start: number;
|
|
411
|
+
end: number;
|
|
247
412
|
label: string;
|
|
248
|
-
|
|
413
|
+
text: string;
|
|
414
|
+
score: number;
|
|
415
|
+
source: string;
|
|
416
|
+
sourceDetail?: string | null;
|
|
249
417
|
};
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
*/
|
|
256
|
-
type CustomRegexPattern = {
|
|
257
|
-
pattern: string;
|
|
258
|
-
label: string;
|
|
259
|
-
score?: number;
|
|
418
|
+
type NativeBindingRedactionResult = {
|
|
419
|
+
redactedText: string;
|
|
420
|
+
redactionMap: NativeBindingRedactionEntry[];
|
|
421
|
+
operatorMap: NativeBindingOperatorEntry[];
|
|
422
|
+
entityCount: number;
|
|
260
423
|
};
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
* Each value is the array of terms for that
|
|
294
|
-
* dictionary.
|
|
295
|
-
*/
|
|
296
|
-
denyList?: Readonly<Record<string, readonly string[]>>;
|
|
297
|
-
/**
|
|
298
|
-
* Metadata per dictionary ID. Required when
|
|
299
|
-
* `denyList` is provided so the pipeline knows
|
|
300
|
-
* labels, categories, and country filters.
|
|
301
|
-
*/
|
|
302
|
-
denyListMeta?: Readonly<Record<string, DictionaryMeta>>;
|
|
303
|
-
/**
|
|
304
|
-
* Pre-loaded city names, already merged across
|
|
305
|
-
* all desired countries.
|
|
306
|
-
*
|
|
307
|
-
* Prefer `citiesByCountry` when callers also pass
|
|
308
|
-
* `denyListCountries` / `denyListRegions`; merged
|
|
309
|
-
* city arrays cannot be scoped after injection.
|
|
310
|
-
*/
|
|
311
|
-
cities?: readonly string[];
|
|
312
|
-
/**
|
|
313
|
-
* Pre-loaded city names keyed by ISO 3166-1 alpha-2
|
|
314
|
-
* country code. When provided, the deny-list builder
|
|
315
|
-
* applies `denyListCountries` / `denyListRegions`
|
|
316
|
-
* before adding city patterns to the search automaton.
|
|
317
|
-
*/
|
|
318
|
-
citiesByCountry?: Readonly<Record<string, readonly string[]>>;
|
|
424
|
+
type NativeBindingStaticRedactionResult = {
|
|
425
|
+
resolvedEntities: NativeBindingPipelineEntity[];
|
|
426
|
+
redaction: NativeBindingRedactionResult;
|
|
427
|
+
};
|
|
428
|
+
type NativePreparedSearchBinding = {
|
|
429
|
+
prepareDiagnosticsJson?: () => string;
|
|
430
|
+
warmLazyRegex?: () => void;
|
|
431
|
+
warm_lazy_regex?: () => void;
|
|
432
|
+
warmLazyRegexDiagnosticsJson?: () => string;
|
|
433
|
+
warm_lazy_regex_diagnostics_json?: () => string;
|
|
434
|
+
redactStaticEntities: (fullText: string, operators?: NativeBindingOperatorConfig) => NativeBindingStaticRedactionResult;
|
|
435
|
+
redactStaticEntitiesJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
436
|
+
redactStaticEntitiesResultStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onEvent: NativeResultEventCallback) => string;
|
|
437
|
+
redactStaticEntitiesDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
438
|
+
redactStaticEntitiesDiagnosticsStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onBatch: NativeDiagnosticsBatchCallback) => string;
|
|
439
|
+
redactStaticEntitiesSummaryDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
440
|
+
};
|
|
441
|
+
type NativeAnonymizeBinding = {
|
|
442
|
+
normalizeForSearch: (text: string) => string;
|
|
443
|
+
nativePackageVersion: () => string;
|
|
444
|
+
NativePreparedSearch: {
|
|
445
|
+
fromConfigJsonBytes: (configJson: Uint8Array) => NativePreparedSearchBinding;
|
|
446
|
+
fromPreparedPackageBytes: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
447
|
+
fromPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
448
|
+
fromTrustedPreparedPackageBytes?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
449
|
+
fromTrustedPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
450
|
+
};
|
|
451
|
+
prepareStaticSearchPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
452
|
+
prepareStaticSearchCompressedPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
453
|
+
assembleStaticSearchConfigJson?: (pipelineConfigJson: Uint8Array, dictionariesJson?: Uint8Array, gazetteerJson?: Uint8Array) => Uint8Array;
|
|
454
|
+
assembleStaticSearchPackageBytes?: (pipelineConfigJson: Uint8Array, dictionariesJson?: Uint8Array, gazetteerJson?: Uint8Array) => Uint8Array;
|
|
455
|
+
assembleStaticSearchCompressedPackageBytes?: (pipelineConfigJson: Uint8Array, dictionariesJson?: Uint8Array, gazetteerJson?: Uint8Array) => Uint8Array;
|
|
319
456
|
};
|
|
320
|
-
type
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
enableRegex: boolean;
|
|
324
|
-
/**
|
|
325
|
-
* Enables legal-form organization detection.
|
|
326
|
-
* Required for typed callers; legacy untyped
|
|
327
|
-
* callers that omit this field are treated as
|
|
328
|
-
* enabled at runtime for backward compatibility.
|
|
329
|
-
*/
|
|
330
|
-
enableLegalForms: boolean;
|
|
331
|
-
/**
|
|
332
|
-
* Enables first-name/surname/title corpus matching.
|
|
333
|
-
* When deny-list mode is enabled, this also controls
|
|
334
|
-
* whether name-corpus entries are injected into the
|
|
335
|
-
* deny-list search automaton.
|
|
336
|
-
*/
|
|
337
|
-
enableNameCorpus: boolean;
|
|
338
|
-
/**
|
|
339
|
-
* Optional language scope for first-name/surname
|
|
340
|
-
* dictionaries, using the keys present in
|
|
341
|
-
* `dictionaries.firstNames` / `dictionaries.surnames`
|
|
342
|
-
* (for example `["en", "de"]`). When omitted, all
|
|
343
|
-
* injected name languages are used for backward
|
|
344
|
-
* compatibility.
|
|
345
|
-
*/
|
|
346
|
-
nameCorpusLanguages?: string[];
|
|
347
|
-
enableDenyList: boolean;
|
|
348
|
-
denyListCountries?: string[];
|
|
349
|
-
denyListRegions?: string[];
|
|
350
|
-
denyListExcludeCategories?: string[];
|
|
351
|
-
/**
|
|
352
|
-
* Caller-owned exact terms to match through the
|
|
353
|
-
* deny-list layer. Requires `enableDenyList: true`.
|
|
354
|
-
*/
|
|
355
|
-
customDenyList?: readonly CustomDenyListEntry[];
|
|
356
|
-
/**
|
|
357
|
-
* Caller-owned regex detectors. Requires
|
|
358
|
-
* `enableRegex: true`.
|
|
359
|
-
*/
|
|
360
|
-
customRegexes?: readonly CustomRegexPattern[];
|
|
361
|
-
enableGazetteer: boolean;
|
|
362
|
-
/**
|
|
363
|
-
* Detect country names (ISO 3166-1 names, curated
|
|
364
|
-
* aliases, alpha-3 codes). Defaults to true. Names
|
|
365
|
-
* span all manifest languages plus widely-used
|
|
366
|
-
* additions (Dutch, Russian, Chinese, Arabic, etc.).
|
|
367
|
-
*/
|
|
368
|
-
enableCountries?: boolean;
|
|
369
|
-
enableNer: boolean;
|
|
370
|
-
enableConfidenceBoost: boolean;
|
|
371
|
-
enableCoreference: boolean;
|
|
372
|
-
enableZoneClassification?: boolean;
|
|
373
|
-
enableHotwordRules?: boolean;
|
|
374
|
-
/**
|
|
375
|
-
* Requested output labels. An empty array means
|
|
376
|
-
* "do not filter by label" for deterministic
|
|
377
|
-
* detectors; NER falls back to DEFAULT_ENTITY_LABELS.
|
|
378
|
-
*/
|
|
379
|
-
labels: string[];
|
|
380
|
-
workspaceId: string;
|
|
381
|
-
/**
|
|
382
|
-
* Pre-loaded dictionary data for name, deny-list,
|
|
383
|
-
* and city detection. When omitted, dictionary-based
|
|
384
|
-
* detection paths are skipped. Consumers load from
|
|
385
|
-
* the anonymize-data package and pass the data here.
|
|
386
|
-
*/
|
|
387
|
-
dictionaries?: Dictionaries;
|
|
457
|
+
type NativeOperatorConfig = {
|
|
458
|
+
operators?: Record<string, OperatorType>;
|
|
459
|
+
redactString?: string;
|
|
388
460
|
};
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
461
|
+
type NativePipelineEntity = {
|
|
462
|
+
start: number;
|
|
463
|
+
end: number;
|
|
392
464
|
label: string;
|
|
465
|
+
text: string;
|
|
393
466
|
score: number;
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
validatorInput?: (text: string) => string;
|
|
467
|
+
source: string;
|
|
468
|
+
sourceDetail?: string;
|
|
397
469
|
};
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
* Get dynamically built date patterns from
|
|
404
|
-
* date-months.json. Returns a cached promise; the JSON
|
|
405
|
-
* is loaded only once.
|
|
406
|
-
*/
|
|
407
|
-
declare const getDatePatterns: () => Promise<string[]>;
|
|
408
|
-
/** Date pattern metadata (all are score 1 dates). */
|
|
409
|
-
declare const DATE_PATTERN_META: Readonly<RegexMeta>;
|
|
410
|
-
/**
|
|
411
|
-
* Get dynamically built monetary amount patterns from
|
|
412
|
-
* currencies.json. Returns a cached promise; the JSON
|
|
413
|
-
* is loaded only once.
|
|
414
|
-
*/
|
|
415
|
-
declare const getCurrencyPatterns: () => Promise<string[]>;
|
|
416
|
-
/** Currency pattern metadata (score 0.9). */
|
|
417
|
-
declare const CURRENCY_PATTERN_META: Readonly<RegexMeta>;
|
|
418
|
-
/**
|
|
419
|
-
* Process regex matches from the unified search.
|
|
420
|
-
* Receives all matches; filters to the regex slice
|
|
421
|
-
* via sliceStart/sliceEnd. Local index into META is
|
|
422
|
-
* match.pattern - sliceStart.
|
|
423
|
-
*
|
|
424
|
-
* For stdnum-derived patterns (those with a validator
|
|
425
|
-
* in META), the matched text is passed through the
|
|
426
|
-
* validator's validate() method. If validation fails,
|
|
427
|
-
* the match is discarded as a false positive.
|
|
428
|
-
*/
|
|
429
|
-
declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, meta_: readonly RegexMeta[]) => Entity[];
|
|
430
|
-
//#endregion
|
|
431
|
-
//#region src/detectors/deny-list.d.ts
|
|
432
|
-
type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries" | "enableCountries">;
|
|
433
|
-
/**
|
|
434
|
-
* Source tag for each pattern in the automaton.
|
|
435
|
-
* "deny-list" = standard deny list entry
|
|
436
|
-
* "city" = city dictionary entry
|
|
437
|
-
* "custom-deny-list" = caller-owned exact term
|
|
438
|
-
* "first-name" = name corpus first name
|
|
439
|
-
* "surname" = name corpus surname
|
|
440
|
-
* "title" = academic/professional title
|
|
441
|
-
*/
|
|
442
|
-
type PatternSource = "deny-list" | "city" | "custom-deny-list" | "first-name" | "surname" | "title";
|
|
443
|
-
type PatternLabels = string | string[];
|
|
444
|
-
type PatternSources = PatternSource | PatternSource[];
|
|
445
|
-
/**
|
|
446
|
-
* Pre-built deny list data. Constructed once by
|
|
447
|
-
* `buildDenyList`, reused across `processDenyListMatches`
|
|
448
|
-
* calls. Contains PatternEntry[] for the unified builder
|
|
449
|
-
* plus parallel label/source arrays for post-processing.
|
|
450
|
-
*/
|
|
451
|
-
type DenyListData = {
|
|
452
|
-
/**
|
|
453
|
-
* Maps pattern index → entity labels (plural).
|
|
454
|
-
* Same pattern can have multiple labels when it
|
|
455
|
-
* appears in multiple dictionaries (e.g., "Denver"
|
|
456
|
-
* is both a person name and a city name).
|
|
457
|
-
*/
|
|
458
|
-
labels: PatternLabels[]; /** Maps pattern index → labels contributed by custom entries. */
|
|
459
|
-
customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
|
|
460
|
-
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
461
|
-
sources: PatternSources[];
|
|
470
|
+
type NativeRedactionResult = {
|
|
471
|
+
redactedText: string;
|
|
472
|
+
redactionMap: Map<string, string>;
|
|
473
|
+
operatorMap: Map<string, OperatorType>;
|
|
474
|
+
entityCount: number;
|
|
462
475
|
};
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
* The returned data provides PatternEntry[] for the
|
|
467
|
-
* unified builder and parallel arrays for
|
|
468
|
-
* post-processing.
|
|
469
|
-
*
|
|
470
|
-
* Dictionary data is injected via `config.dictionaries`.
|
|
471
|
-
* Returns null if no dictionaries are provided.
|
|
472
|
-
*/
|
|
473
|
-
declare const buildDenyList: (config: DenyListConfig, ctx?: PipelineContext) => Promise<DenyListData | null>;
|
|
474
|
-
/**
|
|
475
|
-
* Ensure all deny-list support data (stopwords, allow
|
|
476
|
-
* list, person stopwords, generic roles) is loaded on
|
|
477
|
-
* the given context. Call this before
|
|
478
|
-
* processDenyListMatches / filterFalsePositives when
|
|
479
|
-
* the search instance was built on a different context
|
|
480
|
-
* (e.g. cachedSearch).
|
|
481
|
-
*/
|
|
482
|
-
declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Dictionaries, nameCorpusLanguages?: readonly string[]) => Promise<void>;
|
|
483
|
-
/**
|
|
484
|
-
* Process deny list matches from the unified search.
|
|
485
|
-
* Receives all matches; filters to the deny list slice
|
|
486
|
-
* via sliceStart/sliceEnd. Local index into data.labels,
|
|
487
|
-
* data.originals, data.sources is match.pattern - sliceStart.
|
|
488
|
-
*
|
|
489
|
-
* Two-pass approach to reduce false positives:
|
|
490
|
-
* 1. Collect all matches (case-insensitive,
|
|
491
|
-
* whole-word via Rust automaton)
|
|
492
|
-
* 2. Require uppercase start in source text
|
|
493
|
-
* 3. For person names, require at least one
|
|
494
|
-
* mid-sentence occurrence to prove proper noun
|
|
495
|
-
* 4. Return all occurrences of validated terms
|
|
496
|
-
*/
|
|
497
|
-
declare const processDenyListMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: DenyListData, ctx?: PipelineContext) => Entity[];
|
|
498
|
-
//#endregion
|
|
499
|
-
//#region src/detectors/countries.d.ts
|
|
500
|
-
/**
|
|
501
|
-
* Pre-built country patterns + parallel label/source
|
|
502
|
-
* metadata. Constructed once and reused across pipeline
|
|
503
|
-
* runs.
|
|
504
|
-
*/
|
|
505
|
-
type CountryData = {
|
|
506
|
-
/** Maps local pattern index to entity label. Always "country". */labels: string[];
|
|
507
|
-
/**
|
|
508
|
-
* Maps local pattern index to the alpha-2 ISO code the
|
|
509
|
-
* pattern resolves to. Used for downstream coreference /
|
|
510
|
-
* placeholder grouping.
|
|
511
|
-
*/
|
|
512
|
-
isoCodes: string[]; /** Maps local pattern index to pattern variant kind. */
|
|
513
|
-
variants: CountryVariant[];
|
|
476
|
+
type NativeStaticRedactionResult = {
|
|
477
|
+
resolvedEntities: NativePipelineEntity[];
|
|
478
|
+
redaction: NativeRedactionResult;
|
|
514
479
|
};
|
|
515
|
-
type
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
start: number;
|
|
520
|
-
end: number;
|
|
480
|
+
type NativeSearchPackageOptions = {
|
|
481
|
+
binding: NativeAnonymizeBinding;
|
|
482
|
+
config: NativePreparedSearchConfig;
|
|
483
|
+
compressed?: boolean;
|
|
521
484
|
};
|
|
522
|
-
type
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
*/
|
|
528
|
-
isFuzzy: boolean[];
|
|
485
|
+
type NativeSearchPackageInput = NativePreparedSearchConfig | string | Uint8Array;
|
|
486
|
+
type SharedNativeSearchPackageOptions = {
|
|
487
|
+
binding: NativeAnonymizeBinding;
|
|
488
|
+
config: NativeSearchPackageInput;
|
|
489
|
+
compressed?: boolean;
|
|
529
490
|
};
|
|
530
|
-
type
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
tsLiterals: TextSearch;
|
|
534
|
-
slices: {
|
|
535
|
-
regex: PatternSlice;
|
|
536
|
-
customRegex: PatternSlice;
|
|
537
|
-
legalForms: PatternSlice;
|
|
538
|
-
triggers: PatternSlice;
|
|
539
|
-
denyList: PatternSlice;
|
|
540
|
-
streetTypes: PatternSlice;
|
|
541
|
-
gazetteer: PatternSlice;
|
|
542
|
-
countries: PatternSlice;
|
|
543
|
-
};
|
|
544
|
-
regexMeta: readonly RegexMeta[];
|
|
545
|
-
customRegexMeta: readonly RegexMeta[];
|
|
546
|
-
triggerRules: readonly TriggerRule[];
|
|
547
|
-
denyListData: DenyListData | null;
|
|
548
|
-
gazetteerData: GazetteerData | null;
|
|
549
|
-
countryData: CountryData | null;
|
|
491
|
+
type SharedNativePreparedPackageOptions = {
|
|
492
|
+
binding: NativeAnonymizeBinding;
|
|
493
|
+
packageBytes: Uint8Array;
|
|
550
494
|
};
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
* shallow copies (spread). Uses position + label so the
|
|
557
|
-
* key is identical for the original object and any
|
|
558
|
-
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
559
|
-
*
|
|
560
|
-
* @deprecated No longer used internally: coref alias
|
|
561
|
-
* links travel on the entities themselves
|
|
562
|
-
* (`corefSourceText`). Kept for API compatibility.
|
|
563
|
-
*/
|
|
564
|
-
declare const corefKey: (e: Entity) => string;
|
|
565
|
-
/**
|
|
566
|
-
* Compiled RegExp pattern used for coreference
|
|
567
|
-
* definition extraction.
|
|
568
|
-
*/
|
|
569
|
-
type DefinitionPattern = {
|
|
570
|
-
pattern: RegExp;
|
|
495
|
+
type SharedNativeRedactTextJsonOptions = {
|
|
496
|
+
binding: NativeAnonymizeBinding;
|
|
497
|
+
config: NativeSearchPackageInput;
|
|
498
|
+
fullText: string;
|
|
499
|
+
operators?: NativeOperatorConfig;
|
|
571
500
|
};
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
*/
|
|
577
|
-
type NameCorpusData = {
|
|
578
|
-
firstNames: ReadonlySet<string>;
|
|
579
|
-
surnames: ReadonlySet<string>;
|
|
580
|
-
titleTokens: ReadonlySet<string>;
|
|
581
|
-
/** Abbreviation-style titles whose trailing dot is
|
|
582
|
-
* part of the title, not a sentence boundary.
|
|
583
|
-
* Contains the lowercase, dot-stripped form
|
|
584
|
-
* (e.g., "dr", "smt", "atty"). */
|
|
585
|
-
titleAbbreviations: ReadonlySet<string>;
|
|
586
|
-
excludedWords: ReadonlySet<string>;
|
|
587
|
-
/** Lowercased common English words. A name chain whose
|
|
588
|
-
* every token is a common word (e.g. "Loan Documents",
|
|
589
|
-
* where "Loan" coincides with a Vietnamese given name)
|
|
590
|
-
* is treated as a common-word phrase, not a person. */
|
|
591
|
-
commonWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
|
|
592
|
-
nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
|
|
593
|
-
excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
|
|
594
|
-
firstNamesList: readonly string[];
|
|
595
|
-
surnamesList: readonly string[];
|
|
596
|
-
titlesList: readonly string[];
|
|
597
|
-
excludedList: readonly string[];
|
|
598
|
-
nonWesternNamesList: readonly string[];
|
|
599
|
-
excludedAllCapsList: readonly string[];
|
|
501
|
+
type SharedNativeRedactTextOptions = SharedNativeRedactTextJsonOptions;
|
|
502
|
+
type SharedNativeDiagnosticsJsonOptions = SharedNativeRedactTextJsonOptions;
|
|
503
|
+
type SharedNativeDiagnosticsStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
504
|
+
onBatch: NativeDiagnosticsBatchCallback;
|
|
600
505
|
};
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
* sequence of runs sharing the same config). Replacing
|
|
604
|
-
* module-level singletons with this object enables
|
|
605
|
-
* concurrent pipelines with different configs and
|
|
606
|
-
* simplifies testing.
|
|
607
|
-
*
|
|
608
|
-
* Each field starts null and is populated lazily on
|
|
609
|
-
* first use by the corresponding loader function.
|
|
610
|
-
*/
|
|
611
|
-
type PipelineContext = {
|
|
612
|
-
search: UnifiedSearchInstance | null;
|
|
613
|
-
searchKey: string;
|
|
614
|
-
searchPromise: Promise<UnifiedSearchInstance> | null;
|
|
615
|
-
nameCorpus: NameCorpusData | null;
|
|
616
|
-
nameCorpusKey: string;
|
|
617
|
-
nameCorpusPromise: Promise<void> | null;
|
|
618
|
-
stopwords: ReadonlySet<string> | null;
|
|
619
|
-
stopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
620
|
-
allowList: ReadonlySet<string> | null;
|
|
621
|
-
allowListPromise: Promise<ReadonlySet<string>> | null;
|
|
622
|
-
personStopwords: ReadonlySet<string> | null;
|
|
623
|
-
personStopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
624
|
-
addressStopwords: ReadonlySet<string> | null;
|
|
625
|
-
addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
|
|
626
|
-
firstNameExclusions: ReadonlySet<string> | null;
|
|
627
|
-
firstNameExclusionCorpusLen: number;
|
|
628
|
-
genericRoles: ReadonlySet<string> | null;
|
|
629
|
-
genericRolesPromise: Promise<ReadonlySet<string>> | null;
|
|
630
|
-
corefPatterns: DefinitionPattern[] | null;
|
|
631
|
-
corefPatternsPromise: Promise<DefinitionPattern[]> | null;
|
|
632
|
-
corefLoadAttempted: boolean;
|
|
633
|
-
roleStopSet: ReadonlySet<string> | null;
|
|
634
|
-
roleStopSetPromise: Promise<ReadonlySet<string>> | null;
|
|
635
|
-
zoneHeadingPatterns: RegExp[] | null;
|
|
636
|
-
zoneSigningPatterns: RegExp[] | null;
|
|
637
|
-
zoneInitPromise: Promise<void> | null;
|
|
506
|
+
type SharedNativeRedactTextStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
507
|
+
onEvent: NativeResultEventCallback;
|
|
638
508
|
};
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
//#region src/pipeline.d.ts
|
|
643
|
-
/** Strip leading/trailing whitespace and punctuation. */
|
|
644
|
-
declare const sanitizeEntities: (entities: Entity[]) => Entity[];
|
|
645
|
-
declare const mergeAndDedup: (...layers: Entity[][]) => Entity[];
|
|
646
|
-
type NerInferenceFn = (fullText: string, labels: string[], threshold: number, signal?: AbortSignal) => Promise<Entity[]>;
|
|
647
|
-
type PipelineSearchOptions = {
|
|
648
|
-
config: PipelineConfig;
|
|
649
|
-
gazetteerEntries?: GazetteerEntry[];
|
|
650
|
-
context?: PipelineContext;
|
|
509
|
+
type NativeNormalizeOptions = {
|
|
510
|
+
binding: NativeAnonymizeBinding;
|
|
511
|
+
text: string;
|
|
651
512
|
};
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
513
|
+
type NativeAnonymizerFromConfigOptions = {
|
|
514
|
+
binding: NativeAnonymizeBinding;
|
|
515
|
+
config: NativePreparedSearchConfig;
|
|
516
|
+
};
|
|
517
|
+
type NativeAnonymizerFromPackageOptions = {
|
|
518
|
+
binding: NativeAnonymizeBinding;
|
|
519
|
+
packageBytes: Uint8Array;
|
|
520
|
+
};
|
|
521
|
+
type NativePipelineFromPackageOptions = NativeAnonymizerFromPackageOptions;
|
|
522
|
+
type NativeBindingVersionOptions = {
|
|
523
|
+
binding: NativeAnonymizeBinding;
|
|
524
|
+
expectedVersion: string;
|
|
525
|
+
};
|
|
526
|
+
declare class PreparedNativeAnonymizer {
|
|
527
|
+
#private;
|
|
528
|
+
constructor(prepared: NativePreparedSearchBinding);
|
|
529
|
+
prepareDiagnosticsJson(): string | null;
|
|
530
|
+
prepare_diagnostics_json(): string | null;
|
|
531
|
+
warmLazyRegex(): void;
|
|
532
|
+
warm_lazy_regex(): void;
|
|
533
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
534
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
535
|
+
redactStaticEntities(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
536
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
537
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
538
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
539
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
540
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
541
|
+
redactStaticEntitiesDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
542
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
543
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
544
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
545
|
+
redactStaticEntitiesSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
546
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
547
|
+
}
|
|
548
|
+
declare class PreparedNativePipeline {
|
|
549
|
+
#private;
|
|
550
|
+
constructor(anonymizer: PreparedNativeAnonymizer);
|
|
551
|
+
prepareDiagnosticsJson(): string | null;
|
|
552
|
+
prepare_diagnostics_json(): string | null;
|
|
553
|
+
warmLazyRegex(): void;
|
|
554
|
+
warm_lazy_regex(): void;
|
|
555
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
556
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
557
|
+
redactText(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
558
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
559
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
560
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
561
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
562
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
563
|
+
redactTextDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
564
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
565
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
566
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
567
|
+
redactTextSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
568
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
569
|
+
}
|
|
570
|
+
declare const encodeNativeSearchConfig: (config: NativePreparedSearchConfig) => Uint8Array;
|
|
571
|
+
declare const encodeNativeSearchConfigInput: (config: NativeSearchPackageInput) => Uint8Array;
|
|
572
|
+
declare const getNativeBindingVersion: (binding: NativeAnonymizeBinding) => string;
|
|
573
|
+
declare const assertNativeBindingVersion: ({
|
|
574
|
+
binding,
|
|
575
|
+
expectedVersion
|
|
576
|
+
}: NativeBindingVersionOptions) => void;
|
|
577
|
+
declare const prepareNativeSearchPackage: ({
|
|
578
|
+
binding,
|
|
659
579
|
config,
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
signal?: AbortSignal;
|
|
679
|
-
context?: PipelineContext;
|
|
680
|
-
};
|
|
681
|
-
/**
|
|
682
|
-
* Run the full detection pipeline.
|
|
683
|
-
*
|
|
684
|
-
* Two TextSearch instances scan the text (regex +
|
|
685
|
-
* literals). Results are dispatched to each
|
|
686
|
-
* detector's post-processor by pattern index range.
|
|
687
|
-
*
|
|
688
|
-
* Pass an AbortSignal to cancel the pipeline between
|
|
689
|
-
* stages. Throws a DOMException with name "AbortError"
|
|
690
|
-
* when cancelled.
|
|
691
|
-
*
|
|
692
|
-
* Pass an optional `context` to isolate cached state
|
|
693
|
-
* from other pipeline runs. If omitted, a module-level
|
|
694
|
-
* default context is used (backward compatible).
|
|
695
|
-
*/
|
|
696
|
-
declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
|
|
580
|
+
compressed
|
|
581
|
+
}: NativeSearchPackageOptions) => Uint8Array;
|
|
582
|
+
declare const createNativeAnonymizerFromConfig: ({
|
|
583
|
+
binding,
|
|
584
|
+
config
|
|
585
|
+
}: NativeAnonymizerFromConfigOptions) => PreparedNativeAnonymizer;
|
|
586
|
+
declare const createNativeAnonymizerFromPackage: ({
|
|
587
|
+
binding,
|
|
588
|
+
packageBytes
|
|
589
|
+
}: NativeAnonymizerFromPackageOptions) => PreparedNativeAnonymizer;
|
|
590
|
+
declare const createNativePipelineFromPackage: ({
|
|
591
|
+
binding,
|
|
592
|
+
packageBytes
|
|
593
|
+
}: NativePipelineFromPackageOptions) => PreparedNativePipeline;
|
|
594
|
+
declare const PreparedSearch: typeof PreparedNativeAnonymizer;
|
|
595
|
+
type PreparedSearch = PreparedNativeAnonymizer;
|
|
596
|
+
declare const PreparedAnonymizer: typeof PreparedNativeAnonymizer;
|
|
597
|
+
type PreparedAnonymizer = PreparedNativeAnonymizer;
|
|
697
598
|
//#endregion
|
|
698
599
|
//#region src/redact.d.ts
|
|
699
|
-
/**
|
|
700
|
-
* Build a stable mapping from entity text to numbered
|
|
701
|
-
* placeholders. Same real-world value always maps to the
|
|
702
|
-
* same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
|
|
703
|
-
* both become [PERSON_1]).
|
|
704
|
-
*
|
|
705
|
-
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
706
|
-
* and N is a 1-based counter per label.
|
|
707
|
-
*
|
|
708
|
-
* @param _ctx Unused. Kept for signature compatibility;
|
|
709
|
-
* coref alias links now travel on the entities
|
|
710
|
-
* themselves (`corefSourceText`).
|
|
711
|
-
*/
|
|
712
|
-
declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext) => Map<string, string>;
|
|
713
|
-
/**
|
|
714
|
-
* Apply redactions to the source text, replacing each
|
|
715
|
-
* confirmed entity span using the configured operator.
|
|
716
|
-
*
|
|
717
|
-
* Co-references are consistent: if the same text appears
|
|
718
|
-
* multiple times, all occurrences get the same placeholder.
|
|
719
|
-
*
|
|
720
|
-
* @param ctx Pipeline context. Must be the same instance
|
|
721
|
-
* passed to `runPipeline` (or `findCoreferenceSpans`)
|
|
722
|
-
* so coreference placeholder links are preserved.
|
|
723
|
-
* Defaults to `defaultContext` for single-tenant usage.
|
|
724
|
-
*/
|
|
725
|
-
declare const redactText: (fullText: string, entities: Entity[], config?: OperatorConfig, ctx?: PipelineContext) => RedactionResult;
|
|
726
600
|
/**
|
|
727
601
|
* Serialize the redaction key to JSON for export.
|
|
728
602
|
* Includes operator metadata so the export is self-describing.
|
|
@@ -735,483 +609,60 @@ declare const exportRedactionKey: (redactionMap: Map<string, string>, operatorMa
|
|
|
735
609
|
*/
|
|
736
610
|
declare const deanonymise: (redactedText: string, redactionMap: Map<string, string>) => string;
|
|
737
611
|
//#endregion
|
|
738
|
-
//#region src/
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
*
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
/**
|
|
749
|
-
*
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
*
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
*
|
|
764
|
-
*
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
*
|
|
768
|
-
*
|
|
769
|
-
*
|
|
770
|
-
* `
|
|
771
|
-
*
|
|
772
|
-
*
|
|
773
|
-
*
|
|
774
|
-
|
|
775
|
-
declare const
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
declare const processTriggerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, rules: readonly TriggerRule[]) => Entity[];
|
|
793
|
-
//#endregion
|
|
794
|
-
//#region src/detectors/address-seeds.d.ts
|
|
795
|
-
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
796
|
-
/**
|
|
797
|
-
* Process address seeds from the unified search.
|
|
798
|
-
* Receives all matches; filters to the street types
|
|
799
|
-
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
800
|
-
* existingEntities for seed collection, clustering,
|
|
801
|
-
* expansion, and scoring.
|
|
802
|
-
*
|
|
803
|
-
* Runs as a post-processor after all other detectors,
|
|
804
|
-
* using their output as seed sources.
|
|
805
|
-
*/
|
|
806
|
-
declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
|
|
807
|
-
//#endregion
|
|
808
|
-
//#region src/detectors/gazetteer.d.ts
|
|
809
|
-
/**
|
|
810
|
-
* Build TextSearch-compatible patterns from gazetteer
|
|
811
|
-
* entries. Returns:
|
|
812
|
-
* - Exact literal patterns for all terms
|
|
813
|
-
* - Fuzzy patterns (distance: 2) for terms >= 4 chars
|
|
814
|
-
* - Parallel metadata arrays for post-processing
|
|
815
|
-
*
|
|
816
|
-
* Patterns are ordered: all exact first, then all
|
|
817
|
-
* fuzzy. The isFuzzy array marks which are which.
|
|
818
|
-
*/
|
|
819
|
-
declare const buildGazetteerPatterns: (entries: GazetteerEntry[]) => {
|
|
820
|
-
patterns: PatternEntry[];
|
|
821
|
-
data: GazetteerData;
|
|
822
|
-
};
|
|
823
|
-
/**
|
|
824
|
-
* Process gazetteer matches from the unified literal
|
|
825
|
-
* search. Receives all matches; filters to the
|
|
826
|
-
* gazetteer slice via sliceStart/sliceEnd.
|
|
827
|
-
*
|
|
828
|
-
* Exact matches get score 0.9; fuzzy matches get
|
|
829
|
-
* 0.85. Fuzzy matches that overlap an exact match
|
|
830
|
-
* are dropped.
|
|
831
|
-
*
|
|
832
|
-
* For exact matches, attempts prefix extension for
|
|
833
|
-
* legal suffixes ("a.s.", "GmbH", "s.r.o." after
|
|
834
|
-
* the matched term).
|
|
835
|
-
*/
|
|
836
|
-
declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: GazetteerData) => Entity[];
|
|
837
|
-
//#endregion
|
|
838
|
-
//#region src/detectors/coreference.d.ts
|
|
839
|
-
type DefinedTerm = {
|
|
840
|
-
alias: string;
|
|
841
|
-
label: string; /** Position of the definition in the source text */
|
|
842
|
-
definitionStart: number; /** Original entity text the alias refers to */
|
|
843
|
-
sourceText: string;
|
|
844
|
-
};
|
|
845
|
-
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext) => Promise<DefinedTerm[]>;
|
|
846
|
-
/**
|
|
847
|
-
* Find all occurrences of defined-term aliases in the
|
|
848
|
-
* full text. Returns Entity spans for each match.
|
|
849
|
-
*
|
|
850
|
-
* Respects word boundaries: "Kupující" must not match
|
|
851
|
-
* inside "Kupujícímu". A match is valid only if the
|
|
852
|
-
* character before the start and after the end are NOT
|
|
853
|
-
* word characters (letter/digit).
|
|
854
|
-
*
|
|
855
|
-
* Each returned alias carries `corefSourceText` linking
|
|
856
|
-
* it to its source entity text, for consistent
|
|
857
|
-
* placeholder numbering.
|
|
858
|
-
*
|
|
859
|
-
* @param _ctx Unused. Kept for signature compatibility;
|
|
860
|
-
* alias links now travel on the entities themselves.
|
|
861
|
-
*/
|
|
862
|
-
declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ctx?: PipelineContext) => Entity[];
|
|
863
|
-
//#endregion
|
|
864
|
-
//#region src/detectors/org-propagation.d.ts
|
|
865
|
-
/**
|
|
866
|
-
* After the main detection pass, collect organization
|
|
867
|
-
* entities with a legal form suffix, strip the suffix
|
|
868
|
-
* to get the base name, and re-scan the full text for
|
|
869
|
-
* bare mentions of that base name. Returns new entities
|
|
870
|
-
* for occurrences not already covered.
|
|
871
|
-
*
|
|
872
|
-
* Propagated mentions are coref aliases: each carries
|
|
873
|
-
* `corefSourceText` linking it to the full seed entity
|
|
874
|
-
* text, so placeholder numbering assigns the bare
|
|
875
|
-
* mention the same placeholder as its source ("Acme"
|
|
876
|
-
* and "Acme Corp." both become [ORGANIZATION_1]).
|
|
877
|
-
*/
|
|
878
|
-
declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
|
|
879
|
-
//#endregion
|
|
880
|
-
//#region src/detectors/names.d.ts
|
|
881
|
-
declare const getNameCorpusNonWesternNames: (ctx?: PipelineContext) => readonly string[];
|
|
882
|
-
/**
|
|
883
|
-
* Load name corpus data from injected dictionaries
|
|
884
|
-
* and legacy config files. Merges all sources.
|
|
885
|
-
*
|
|
886
|
-
* Safe to call multiple times; only loads once per
|
|
887
|
-
* context. Must be called before detectNameCorpus or
|
|
888
|
-
* the getNameCorpus*() accessors are used.
|
|
889
|
-
*
|
|
890
|
-
* @param dictionaries Optional pre-loaded dictionaries
|
|
891
|
-
* with per-language first names and surnames. When
|
|
892
|
-
* omitted, only legacy config files are used.
|
|
893
|
-
*/
|
|
894
|
-
declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionaries, languages?: readonly string[]) => Promise<void>;
|
|
895
|
-
type NameCorpusDetectionOptions = {
|
|
896
|
-
mode?: "full" | "supplemental";
|
|
897
|
-
};
|
|
898
|
-
/**
|
|
899
|
-
* Detect person names by looking up tokens against the
|
|
900
|
-
* name corpus, then chaining adjacent name-like tokens.
|
|
901
|
-
* Handles both Western and non-Western name patterns.
|
|
902
|
-
*
|
|
903
|
-
* Requires initNameCorpus() to have been called first.
|
|
904
|
-
* If not initialized, returns an empty array.
|
|
905
|
-
*
|
|
906
|
-
* Scoring (Western):
|
|
907
|
-
* TITLE + NAME/SURNAME → 0.95
|
|
908
|
-
* NAME + NAME/SURNAME → 0.9
|
|
909
|
-
* SURNAME + NAME/SURNAME → 0.9
|
|
910
|
-
* NAME + CAPITALIZED → 0.7
|
|
911
|
-
* ABBREVIATION + NAME → 0.7
|
|
912
|
-
* Standalone NAME → 0.5 (low confidence)
|
|
913
|
-
* Standalone SURNAME → skip (too ambiguous)
|
|
914
|
-
*
|
|
915
|
-
* Scoring (non-Western, when chain contains nonWestern tokens):
|
|
916
|
-
* TITLE + (nonWestern|CAPITALIZED) → 0.95
|
|
917
|
-
* JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
|
|
918
|
-
* ARABIC_CONNECTOR + nonWestern → 0.9
|
|
919
|
-
* 2+ nonWestern tokens → 0.9
|
|
920
|
-
* nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
|
|
921
|
-
* Standalone nonWestern mid-sentence → 0.5
|
|
922
|
-
*/
|
|
923
|
-
declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext, options?: NameCorpusDetectionOptions) => Entity[];
|
|
924
|
-
//#endregion
|
|
925
|
-
//#region src/unified-search.d.ts
|
|
926
|
-
type UnifiedResult = {
|
|
927
|
-
/** All matches from both instances combined. */regexMatches: Match[];
|
|
928
|
-
customRegexMatches: Match[];
|
|
929
|
-
literalMatches: Match[];
|
|
930
|
-
};
|
|
931
|
-
declare const runUnifiedSearch: (instance: UnifiedSearchInstance, fullText: string) => UnifiedResult;
|
|
932
|
-
//#endregion
|
|
933
|
-
//#region src/regions.d.ts
|
|
934
|
-
/**
|
|
935
|
-
* Geographic regions and country code mappings for
|
|
936
|
-
* scoping deny list dictionaries.
|
|
937
|
-
*/
|
|
938
|
-
declare const REGIONS: {
|
|
939
|
-
readonly Global: null;
|
|
940
|
-
readonly International: null;
|
|
941
|
-
readonly Europe: readonly ["AL", "AD", "AT", "BE", "BA", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IS", "IE", "IT", "XK", "LV", "LI", "LT", "LU", "MD", "ME", "MK", "MT", "MC", "NL", "NO", "PL", "PT", "RO", "RS", "SK", "SI", "ES", "SE", "CH", "UA", "GB"];
|
|
942
|
-
readonly Americas: readonly ["US", "CA", "MX", "BR", "AR", "CL", "CO", "PE", "EC", "VE", "UY", "PY", "BO", "CR", "PA", "DO", "GT", "HN", "SV", "NI", "CU"];
|
|
943
|
-
readonly AsiaPacific: readonly ["AU", "NZ", "JP", "KR", "CN", "TW", "SG", "MY", "TH", "VN", "PH", "ID", "IN", "PK", "BD", "LK", "NP", "HK", "MO"];
|
|
944
|
-
readonly MENA: readonly ["AE", "SA", "IL", "TR", "EG", "JO", "LB", "IQ", "IR", "QA", "KW", "BH", "OM", "MA", "TN", "DZ", "LY", "SY", "YE", "PS"];
|
|
945
|
-
readonly SubSaharanAfrica: readonly ["ZA", "NG", "KE", "GH", "TZ", "ET", "SN", "CI", "CM", "UG", "RW", "MZ", "AO", "ZW", "BW", "NA", "MU"];
|
|
946
|
-
readonly EU: readonly ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE"];
|
|
947
|
-
readonly DACH: readonly ["DE", "AT", "CH"];
|
|
948
|
-
readonly Nordics: readonly ["DK", "SE", "NO", "FI", "IS"];
|
|
949
|
-
readonly CEE: readonly ["CZ", "SK", "PL", "HU", "RO", "BG", "HR", "SI", "LT", "LV", "EE"];
|
|
950
|
-
readonly Anglosphere: readonly ["GB", "US", "CA", "AU", "NZ", "IE"];
|
|
951
|
-
readonly Benelux: readonly ["BE", "NL", "LU"];
|
|
952
|
-
readonly GulfStates: readonly ["AE", "SA", "QA", "KW", "BH", "OM"];
|
|
953
|
-
readonly SouthAsia: readonly ["IN", "PK", "BD", "LK", "NP"];
|
|
954
|
-
readonly EastAsia: readonly ["CN", "JP", "KR", "TW"];
|
|
955
|
-
readonly SoutheastAsia: readonly ["SG", "MY", "TH", "VN", "PH", "ID"];
|
|
956
|
-
readonly Oceania: readonly ["AU", "NZ"];
|
|
957
|
-
};
|
|
958
|
-
type RegionId = keyof typeof REGIONS;
|
|
959
|
-
type RegionArrays = { [K in RegionId]: (typeof REGIONS)[K] };
|
|
960
|
-
type NonNullRegion = { [K in RegionId as RegionArrays[K] extends null ? never : K]: RegionArrays[K] };
|
|
961
|
-
type CountryCode = NonNullRegion[keyof NonNullRegion][number];
|
|
962
|
-
/**
|
|
963
|
-
* Expand region names to country codes and merge with
|
|
964
|
-
* explicit country codes. Returns null when both inputs
|
|
965
|
-
* are empty/undefined (meaning "match all countries").
|
|
966
|
-
*/
|
|
967
|
-
declare const resolveCountries: (regions?: string[], countries?: string[]) => Set<string> | null;
|
|
968
|
-
//#endregion
|
|
969
|
-
//#region src/filters/false-positives.d.ts
|
|
970
|
-
/** Ensure street-type vocabulary is loaded. */
|
|
971
|
-
declare const initAddressComponents: () => Promise<void>;
|
|
972
|
-
/**
|
|
973
|
-
* Filter out entities that are likely false positives:
|
|
974
|
-
* template placeholders, clause/section numbers,
|
|
975
|
-
* standalone years, and generic legal role terms.
|
|
976
|
-
*
|
|
977
|
-
* Runs as a post-processing step after all detection
|
|
978
|
-
* layers have merged.
|
|
979
|
-
*/
|
|
980
|
-
declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext, fullText?: string) => Entity[];
|
|
981
|
-
//#endregion
|
|
982
|
-
//#region src/filters/confidence-boost.d.ts
|
|
983
|
-
/**
|
|
984
|
-
* Boost confidence of near-miss NER entities that appear
|
|
985
|
-
* near high-confidence detections (regex, trigger phrase).
|
|
986
|
-
*
|
|
987
|
-
* If an NER entity scored between (threshold - 0.15) and
|
|
988
|
-
* threshold, count how many confirmed entities exist within
|
|
989
|
-
* a 150-char window. Add +0.05 per co-located entity.
|
|
990
|
-
* If the boosted score crosses the threshold, include it.
|
|
991
|
-
*
|
|
992
|
-
* Only mutates score on near-miss entities; high-confidence
|
|
993
|
-
* entities pass through unchanged.
|
|
994
|
-
*/
|
|
995
|
-
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
996
|
-
//#endregion
|
|
997
|
-
//#region src/filters/hotword-rules.d.ts
|
|
998
|
-
type HotwordRule = {
|
|
999
|
-
hotwords: string[];
|
|
1000
|
-
targetLabels: string[];
|
|
1001
|
-
scoreAdjustment: number;
|
|
1002
|
-
reclassifyTo?: string;
|
|
1003
|
-
proximityBefore: number;
|
|
1004
|
-
proximityAfter: number;
|
|
1005
|
-
};
|
|
1006
|
-
/**
|
|
1007
|
-
* Load hotword rules from the data package.
|
|
1008
|
-
* Safe to call multiple times; subsequent calls
|
|
1009
|
-
* are no-ops.
|
|
1010
|
-
*/
|
|
1011
|
-
declare const initHotwordRules: () => Promise<void>;
|
|
1012
|
-
/**
|
|
1013
|
-
* Apply hotword context rules to detected entities.
|
|
1014
|
-
*
|
|
1015
|
-
* Scans `fullText` once with a single AC automaton
|
|
1016
|
-
* for all hotwords across all rules, then checks
|
|
1017
|
-
* proximity to each entity. Distance-decayed
|
|
1018
|
-
* adjustment: closer hotwords give a stronger boost.
|
|
1019
|
-
*
|
|
1020
|
-
* Returns a new array; input entities are not mutated.
|
|
1021
|
-
*/
|
|
1022
|
-
declare const applyHotwordRules: (entities: Entity[], fullText: string) => Entity[];
|
|
1023
|
-
//#endregion
|
|
1024
|
-
//#region src/filters/zone-classifier.d.ts
|
|
1025
|
-
type DocumentZone = "header" | "signature" | "body" | "table";
|
|
1026
|
-
type ZoneSpan = {
|
|
1027
|
-
zone: DocumentZone;
|
|
1028
|
-
start: number;
|
|
1029
|
-
end: number;
|
|
1030
|
-
};
|
|
1031
|
-
/**
|
|
1032
|
-
* Additive score adjustments per document zone.
|
|
1033
|
-
* Header and signature blocks are dense with PII;
|
|
1034
|
-
* tables often contain structured identifying data.
|
|
1035
|
-
*/
|
|
1036
|
-
declare const ZONE_SCORE_ADJUSTMENTS: {
|
|
1037
|
-
readonly header: 0.1;
|
|
1038
|
-
readonly signature: 0.15;
|
|
1039
|
-
readonly body: 0;
|
|
1040
|
-
readonly table: 0.05;
|
|
1041
|
-
};
|
|
1042
|
-
/**
|
|
1043
|
-
* Ensure config data is loaded. Call once before
|
|
1044
|
-
* classifyZones. Safe to call multiple times.
|
|
1045
|
-
*/
|
|
1046
|
-
declare const initZoneClassifier: (ctx?: PipelineContext) => Promise<void>;
|
|
1047
|
-
/**
|
|
1048
|
-
* Classify a document into zones based on
|
|
1049
|
-
* structural heuristics. Zones are non-overlapping
|
|
1050
|
-
* and cover the entire text.
|
|
1051
|
-
*
|
|
1052
|
-
* Must call `initZoneClassifier()` first.
|
|
1053
|
-
*/
|
|
1054
|
-
declare const classifyZones: (fullText: string, ctx?: PipelineContext) => ZoneSpan[];
|
|
1055
|
-
/**
|
|
1056
|
-
* Apply zone-based score adjustments to entities.
|
|
1057
|
-
* Entities in header/signature/table zones get a
|
|
1058
|
-
* small additive boost reflecting the higher PII
|
|
1059
|
-
* density in those regions.
|
|
1060
|
-
*
|
|
1061
|
-
* Returns a new array; does not mutate inputs.
|
|
1062
|
-
*/
|
|
1063
|
-
declare const applyZoneAdjustments: (entities: Entity[], zones: ZoneSpan[]) => Entity[];
|
|
1064
|
-
//#endregion
|
|
1065
|
-
//#region src/gliner/types.d.ts
|
|
1066
|
-
/**
|
|
1067
|
-
* GLiNER inference types.
|
|
1068
|
-
*
|
|
1069
|
-
* Forked from gliner@0.0.19 (MIT), stripped to runtime-
|
|
1070
|
-
* agnostic core. Original: github.com/Ingvarstep/GLiNER.js
|
|
1071
|
-
*/
|
|
1072
|
-
type EntityResult = {
|
|
1073
|
-
spanText: string;
|
|
1074
|
-
start: number;
|
|
1075
|
-
end: number;
|
|
1076
|
-
label: string;
|
|
1077
|
-
score: number;
|
|
1078
|
-
};
|
|
1079
|
-
/**
|
|
1080
|
-
* Raw inference output: per-batch array of
|
|
1081
|
-
* [spanText, start, end, label, score] tuples.
|
|
1082
|
-
*/
|
|
1083
|
-
type RawInferenceResult = [string, number, number, string, number][][];
|
|
1084
|
-
//#endregion
|
|
1085
|
-
//#region src/gliner/decoder.d.ts
|
|
1086
|
-
/**
|
|
1087
|
-
* Decode span-level model logits into entity results.
|
|
1088
|
-
*/
|
|
1089
|
-
declare const decodeSpans: (batchSize: number, inputLength: number, maxWidth: number, numEntities: number, texts: string[], batchIds: number[], batchWordsStartIdx: number[][], batchWordsEndIdx: number[][], idToClass: Record<number, string>, modelOutput: ArrayLike<number>, flatNer: boolean, threshold: number, multiLabel: boolean) => RawInferenceResult;
|
|
1090
|
-
//#endregion
|
|
1091
|
-
//#region src/gliner/token-decoder.d.ts
|
|
1092
|
-
/**
|
|
1093
|
-
* Decode token-level BIO logits into entity spans.
|
|
1094
|
-
*
|
|
1095
|
-
* For each word, checks if the B(egin) logit for any class
|
|
1096
|
-
* exceeds the threshold. If so, extends the span by consuming
|
|
1097
|
-
* subsequent I(nside) tokens of the same class.
|
|
1098
|
-
*/
|
|
1099
|
-
declare const decodeTokenSpans: (batchSize: number, numWords: number, numEntities: number, texts: string[], batchIds: number[], batchWordsStartIdx: number[][], batchWordsEndIdx: number[][], idToClass: Record<number, string>, modelOutput: ArrayLike<number>, threshold: number) => RawInferenceResult;
|
|
1100
|
-
//#endregion
|
|
1101
|
-
//#region src/gliner/processor.d.ts
|
|
1102
|
-
/** Tokenize text into words with character offsets. */
|
|
1103
|
-
declare const tokenizeText: (text: string) => [words: string[], starts: number[], ends: number[]];
|
|
1104
|
-
/** Prepare a complete batch for ONNX inference. */
|
|
1105
|
-
declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: string[], maxWidth: number) => {
|
|
1106
|
-
inputsIds: number[][];
|
|
1107
|
-
attentionMasks: number[][];
|
|
1108
|
-
wordsMasks: number[][];
|
|
1109
|
-
textLengths: number[];
|
|
1110
|
-
spanIdxs: number[][][];
|
|
1111
|
-
spanMasks: boolean[][];
|
|
1112
|
-
idToClass: Record<number, string>;
|
|
1113
|
-
batchTokens: string[][];
|
|
1114
|
-
batchWordsStartIdx: number[][];
|
|
1115
|
-
batchWordsEndIdx: number[][];
|
|
1116
|
-
};
|
|
1117
|
-
//#endregion
|
|
1118
|
-
//#region src/util/chunker.d.ts
|
|
1119
|
-
/** A chunk paired with its start offset in the source text. */
|
|
1120
|
-
type ChunkSpan = {
|
|
1121
|
-
text: string;
|
|
1122
|
-
offset: number;
|
|
1123
|
-
};
|
|
1124
|
-
/**
|
|
1125
|
-
* Split text into overlapping chunks, each paired with its
|
|
1126
|
-
* exact start offset in the source text.
|
|
1127
|
-
*
|
|
1128
|
-
* Carrying the offset out of the splitter is the robust way to
|
|
1129
|
-
* map chunk-local entity offsets back to document offsets:
|
|
1130
|
-
* downstream code never has to re-locate a chunk by content
|
|
1131
|
-
* search (which mis-locates when boilerplate repeats; see
|
|
1132
|
-
* computeChunkOffsets).
|
|
1133
|
-
*
|
|
1134
|
-
* Character-based splitting (rough token approximation for
|
|
1135
|
-
* GLiNER's ~512 token window); breaks at sentence boundaries
|
|
1136
|
-
* when possible.
|
|
1137
|
-
*/
|
|
1138
|
-
declare const chunkTextWithOffsets: (text: string) => ChunkSpan[];
|
|
1139
|
-
/**
|
|
1140
|
-
* Split text into overlapping chunks for GLiNER's ~512 token
|
|
1141
|
-
* context window. Character-based splitting (rough token
|
|
1142
|
-
* approximation); breaks at sentence boundaries when possible.
|
|
1143
|
-
*
|
|
1144
|
-
* Prefer chunkTextWithOffsets when you also need each chunk's
|
|
1145
|
-
* document offset.
|
|
1146
|
-
*/
|
|
1147
|
-
declare const chunkText: (text: string) => string[];
|
|
1148
|
-
/**
|
|
1149
|
-
* Compute the start offset of each chunk within the original
|
|
1150
|
-
* document text by content search.
|
|
1151
|
-
*
|
|
1152
|
-
* @deprecated Re-locates each chunk with `indexOf`, which can
|
|
1153
|
-
* match the wrong position when identical content repeats in
|
|
1154
|
-
* the document (common in boilerplate-heavy legal text) and
|
|
1155
|
-
* then desyncs every subsequent offset. Use
|
|
1156
|
-
* `chunkTextWithOffsets`, which carries exact offsets out of
|
|
1157
|
-
* the splitter.
|
|
1158
|
-
*/
|
|
1159
|
-
declare const computeChunkOffsets: (fullText: string, chunks: string[]) => number[];
|
|
1160
|
-
/**
|
|
1161
|
-
* Merge entities from overlapping chunks back to
|
|
1162
|
-
* document-level offsets. Deduplicates entities that
|
|
1163
|
-
* appear in overlap regions (keeps highest score).
|
|
1164
|
-
*
|
|
1165
|
-
* Dedup invariant: each incoming entity is compared
|
|
1166
|
-
* against the highest-scored same-label near-dup in
|
|
1167
|
-
* its proximity window. If it loses, it is dropped.
|
|
1168
|
-
* This does NOT guarantee that all pairwise near-dup
|
|
1169
|
-
* relationships in the output are resolved; a lower-
|
|
1170
|
-
* scored entity can survive if the bridging entity
|
|
1171
|
-
* that would have replaced it was itself dropped by
|
|
1172
|
-
* a higher-scored match.
|
|
1173
|
-
*
|
|
1174
|
-
* Uses a reverse-scan over the sorted merged array
|
|
1175
|
-
* so each entity only compares against nearby
|
|
1176
|
-
* predecessors — O(n * w) average where w is the max
|
|
1177
|
-
* entities per POSITION_THRESHOLD window, O(n²) worst
|
|
1178
|
-
* case when replacements dominate (splice is O(n)).
|
|
1179
|
-
*/
|
|
1180
|
-
declare const mergeChunkEntities: (chunkOffsets: number[], chunkResults: Entity[][]) => Entity[];
|
|
1181
|
-
//#endregion
|
|
1182
|
-
//#region src/util/levenshtein.d.ts
|
|
1183
|
-
/**
|
|
1184
|
-
* Compute the Levenshtein edit distance between two
|
|
1185
|
-
* strings. O(n*m) time, O(min(n,m)) space using a
|
|
1186
|
-
* single-row DP approach.
|
|
1187
|
-
*/
|
|
1188
|
-
declare const levenshtein: (rawA: string, rawB: string) => number;
|
|
1189
|
-
//#endregion
|
|
1190
|
-
//#region src/util/normalize.d.ts
|
|
1191
|
-
/**
|
|
1192
|
-
* Normalize typographic variants for search matching.
|
|
1193
|
-
*
|
|
1194
|
-
* Legal documents (especially Czech/German) use
|
|
1195
|
-
* non-breaking spaces, smart quotes, and en/em dashes
|
|
1196
|
-
* that differ from their ASCII equivalents. Since all
|
|
1197
|
-
* replacements are same-length (single code unit →
|
|
1198
|
-
* single code unit), character offsets remain valid.
|
|
1199
|
-
*
|
|
1200
|
-
* Lives here (application layer) rather than in the
|
|
1201
|
-
* AC library: what to normalize is domain-specific.
|
|
1202
|
-
*
|
|
1203
|
-
* Uses a char-code lookup (`Map<number, number>`) and
|
|
1204
|
-
* `Uint16Array` instead of 7 sequential `replaceAll`
|
|
1205
|
-
* calls. For a 50 KB document this eliminates ~350 KB
|
|
1206
|
-
* of intermediate string allocations.
|
|
1207
|
-
*
|
|
1208
|
-
* When no replaceable characters are present (common
|
|
1209
|
-
* for plain-text inputs), a fast-path scan returns the
|
|
1210
|
-
* original string without any allocation. When special
|
|
1211
|
-
* characters exist, the string is scanned twice: once
|
|
1212
|
-
* to detect, once to build the replacement array.
|
|
1213
|
-
*/
|
|
1214
|
-
declare const normalizeForSearch: (text: string) => string;
|
|
612
|
+
//#region src/wasm.d.ts
|
|
613
|
+
/** A prepared package the caller supplies: raw bytes, an ArrayBuffer, or a URL
|
|
614
|
+
* (string or `URL`) that resolves to the package and is fetched. */
|
|
615
|
+
type PreparedPackageSource = Uint8Array | ArrayBuffer | URL | string;
|
|
616
|
+
/** Escape hatch for callers that already hold a binding (e.g. a custom sidecar
|
|
617
|
+
* or a test double). When omitted, the lazily-instantiated wasm binding is
|
|
618
|
+
* used. */
|
|
619
|
+
type WasmBindingOptions = {
|
|
620
|
+
binding?: NativeAnonymizeBinding;
|
|
621
|
+
};
|
|
622
|
+
/** Instantiate (once) and return the wasm binding. Safe to call repeatedly:
|
|
623
|
+
* the underlying wasm module is instantiated a single time and cached. */
|
|
624
|
+
declare const getBinding: () => Promise<NativeAnonymizeBinding>;
|
|
625
|
+
type LoadPreparedPackageOptions = WasmBindingOptions;
|
|
626
|
+
/** Load a prepared package and return a pipeline ready to redact text. */
|
|
627
|
+
declare const loadPipeline: (source: PreparedPackageSource, options?: LoadPreparedPackageOptions) => Promise<PreparedNativePipeline>;
|
|
628
|
+
/** Load a prepared package and return the lower-level anonymizer. */
|
|
629
|
+
declare const load_prepared_package: (source: PreparedPackageSource, options?: LoadPreparedPackageOptions) => Promise<PreparedNativeAnonymizer>;
|
|
630
|
+
/** URL of a bundled default prepared package, resolved against this module so
|
|
631
|
+
* it points at the `native/` asset directory shipped in the tarball. */
|
|
632
|
+
declare const defaultPackageUrl: (language?: string) => URL;
|
|
633
|
+
/** Load a fresh pipeline from the bundled default prepared package.
|
|
634
|
+
*
|
|
635
|
+
* Mirrors the node loader's regional-tag fallback: when an exact package for
|
|
636
|
+
* a locale tag such as `en-US` is not shipped, the base-language package
|
|
637
|
+
* (`en`) is loaded instead. The browser cannot check asset existence up
|
|
638
|
+
* front, so the fallback triggers on a failed load of the exact package. */
|
|
639
|
+
declare const loadDefaultPipeline: (language?: string, options?: LoadPreparedPackageOptions) => Promise<PreparedNativePipeline>;
|
|
640
|
+
/** Cached variant of {@link loadDefaultPipeline}: the default pipeline for a
|
|
641
|
+
* given language is fetched and prepared once, then reused.
|
|
642
|
+
*
|
|
643
|
+
* Only the ambient-binding case is cached. The cache key is language-only, so a
|
|
644
|
+
* caller that injects its own `options.binding` bypasses the cache entirely:
|
|
645
|
+
* reusing a pipeline built against a different binding would be wrong, and
|
|
646
|
+
* folding the binding into the key would keep unbounded per-binding entries
|
|
647
|
+
* alive. Injected-binding callers get a fresh pipeline each call. */
|
|
648
|
+
declare const getDefaultPipeline: (language?: string, options?: LoadPreparedPackageOptions) => Promise<PreparedNativePipeline>;
|
|
649
|
+
declare const redactDefaultText: (fullText: string, operators?: NativeOperatorConfig, language?: string) => Promise<NativeStaticRedactionResult>;
|
|
650
|
+
declare const redactDefaultTextJson: (fullText: string, operators?: NativeOperatorConfig, language?: string) => Promise<string>;
|
|
651
|
+
declare const native_package_version: (options?: WasmBindingOptions) => Promise<string>;
|
|
652
|
+
declare const normalize_for_search: (text: string, options?: WasmBindingOptions) => Promise<string>;
|
|
653
|
+
type PrepareSearchPackageOptions = WasmBindingOptions & {
|
|
654
|
+
compressed?: boolean;
|
|
655
|
+
};
|
|
656
|
+
declare const prepare_search_package: (config: NativeSearchPackageInput, {
|
|
657
|
+
compressed,
|
|
658
|
+
...options
|
|
659
|
+
}?: PrepareSearchPackageOptions) => Promise<Uint8Array>;
|
|
660
|
+
declare const redact_text: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<NativeStaticRedactionResult>;
|
|
661
|
+
declare const redact_text_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string>;
|
|
662
|
+
declare const redact_text_stream_json: (config: NativeSearchPackageInput, fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
663
|
+
declare const diagnostics_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
664
|
+
declare const diagnostics_stream_json: (config: NativeSearchPackageInput, fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
665
|
+
declare const summary_diagnostics_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: WasmBindingOptions) => Promise<string | null>;
|
|
1215
666
|
//#endregion
|
|
1216
|
-
export { type AnonymisationOperator,
|
|
667
|
+
export { type AnonymisationOperator, DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, type DetectionSource, type Entity, LoadPreparedPackageOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeDiagnosticsBatchCallback, NativeNormalizeOptions, NativeOperatorConfig, NativePipelineEntity, NativePipelineFromPackageOptions, NativePreparedSearchBinding, type NativePreparedSearchConfig, NativeRedactionResult, NativeResultEventCallback, NativeSearchPackageInput, NativeSearchPackageOptions, NativeStaticRedactionResult, OPERATOR_TYPES, type OperatorConfig, type OperatorType, PrepareSearchPackageOptions, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedPackageSource, PreparedSearch, type RedactionResult, type ReviewDecision, type ReviewedEntity, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, WasmBindingOptions, assertNativeBindingVersion, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromPackage, deanonymise, defaultPackageUrl, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getBinding, getDefaultPipeline, getNativeBindingVersion, loadDefaultPipeline, loadPipeline, load_prepared_package, native_package_version, normalize_for_search, prepareNativeSearchPackage, prepare_search_package, redactDefaultText, redactDefaultTextJson, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
1217
668
|
//# sourceMappingURL=wasm.d.mts.map
|