@stll/anonymize 1.4.11 → 2.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/ATTRIBUTION.md +70 -0
  2. package/README.md +86 -48
  3. package/dist/address-boundaries.mjs +2 -0
  4. package/dist/address-jurisdiction-prefixes.mjs +16 -0
  5. package/dist/address-jurisdiction-prefixes.mjs.map +1 -0
  6. package/dist/address-stop-keywords.mjs +12 -1
  7. package/dist/address-stopwords.mjs +2 -0
  8. package/dist/address-unit-abbreviations.mjs +15 -0
  9. package/dist/address-unit-abbreviations.mjs.map +1 -0
  10. package/dist/allow-list.mjs +1 -0
  11. package/dist/clause-noun-heads.mjs +5 -1
  12. package/dist/constants.mjs +1 -0
  13. package/dist/constants.mjs.map +1 -1
  14. package/dist/constants2.d.mts +1 -1
  15. package/dist/coreference-org-determiners.mjs +19 -0
  16. package/dist/coreference-org-determiners.mjs.map +1 -0
  17. package/dist/defined-term-heads.mjs +15 -0
  18. package/dist/defined-term-heads.mjs.map +1 -0
  19. package/dist/false-positive-shapes.mjs +36 -0
  20. package/dist/false-positive-shapes.mjs.map +1 -0
  21. package/dist/index.d.mts +3 -1167
  22. package/dist/index.mjs +3 -16079
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/legal-role-heads.cs.mjs +6 -0
  25. package/dist/native-node.d.mts +124 -0
  26. package/dist/native-node.mjs +3 -0
  27. package/dist/native-node2.d.mts +3 -0
  28. package/dist/native-node2.mjs +13353 -0
  29. package/dist/native-node2.mjs.map +1 -0
  30. package/dist/native.d.mts +1158 -0
  31. package/dist/native.mjs +230 -0
  32. package/dist/native.mjs.map +1 -0
  33. package/dist/native2.d.mts +2 -0
  34. package/dist/organization-unit-heads.mjs +20 -0
  35. package/dist/organization-unit-heads.mjs.map +1 -0
  36. package/dist/person-stopwords.mjs +205 -199
  37. package/dist/signing-clauses.mjs +33 -9
  38. package/index.cjs +3 -0
  39. package/native-pipeline.cs.stlanonpkg +0 -0
  40. package/native-pipeline.de.stlanonpkg +0 -0
  41. package/native-pipeline.en.stlanonpkg +0 -0
  42. package/native-pipeline.stlanonpkg +0 -0
  43. package/package.json +42 -8
  44. package/scripts/build-native-pipeline-package.mjs +225 -0
  45. package/dist/address-prepositions.mjs +0 -182
  46. package/dist/address-prepositions.mjs.map +0 -1
package/dist/index.d.mts CHANGED
@@ -1,722 +1,8 @@
1
1
  import { a as OPERATOR_TYPES, i as DetectionSource, n as DETECTION_SOURCES, o as OperatorType, r as DETECTOR_PRIORITY, t as DEFAULT_ENTITY_LABELS } from "./constants2.mjs";
2
- import { Match, PatternEntry, TextSearch } from "@stll/text-search";
3
- import { Validator } from "@stll/stdnum";
4
- import { Tokenizer } from "@huggingface/tokenizers";
2
+ import { $ as Entity, A as createNativePipelineFromPackage, C as SharedNativeRedactTextJsonOptions, D as assertNativeBindingVersion, E as SharedNativeSearchPackageOptions, F as getNativeBindingVersion, J as CustomDenyListEntry, N as encodeNativeSearchConfig, O as createNativeAnonymizerFromConfig, P as encodeNativeSearchConfigInput, Q as DictionaryMeta, S as SharedNativePreparedPackageOptions, T as SharedNativeRedactTextStreamJsonOptions, X as DenyListCategory, Y as CustomRegexPattern, Z as Dictionaries, _ as PreparedNativeAnonymizer, a as NativeDiagnosticsBatchCallback, at as ReviewedEntity, b as SharedNativeDiagnosticsJsonOptions, c as NativePipelineEntity, ct as TriggerRule, d as NativeRedactionResult, et as GazetteerEntry, f as NativeResultEventCallback, g as PreparedAnonymizer, h as NativeStaticRedactionResult, i as NativeBindingVersionOptions, it as ReviewDecision, k as createNativeAnonymizerFromPackage, l as NativePipelineFromPackageOptions, lt as TriggerStrategy, m as NativeSearchPackageOptions, n as NativeAnonymizerFromConfigOptions, nt as PipelineConfig, o as NativeNormalizeOptions, ot as TriggerExtension, p as NativeSearchPackageInput, q as AnonymisationOperator, r as NativeAnonymizerFromPackageOptions, rt as RedactionResult, s as NativeOperatorConfig, st as TriggerGroupConfig, t as NativeAnonymizeBinding, tt as OperatorConfig, u as NativePreparedSearchBinding, ut as TriggerValidation, v as PreparedNativePipeline, w as SharedNativeRedactTextOptions, x as SharedNativeDiagnosticsStreamJsonOptions, y as PreparedSearch, z as prepareNativeSearchPackage } from "./native.mjs";
3
+ import { A as readDefaultNativePipelinePackageFileAsync, B as redact_text_stream_json, C as native_package_version, D as preload_default_native_pipeline, E as preloadDefaultNativePipelineAsync, F as redactDefaultTextJson, G as NativePipelinePackageOptions, H as DEFAULT_NATIVE_PIPELINE_CONFIG, I as redact_default_text, J as createNativePipelineFromConfig, K as NativePipelineUnsupportedFeature, L as redact_default_text_json, M as readNativePipelinePackageFileAsync, N as read_default_native_pipeline_package_file, O as prepare_search_package, P as redactDefaultText, R as redact_text, S as load_prepared_package_file, T as preloadDefaultNativePipeline, U as NativePipelineBuildOptions, V as summary_diagnostics_json, W as NativePipelineCompatibility, X as prepareNativePipelineConfig, Y as getNativePipelineCompatibility, Z as prepareNativePipelinePackage, _ as diagnostics_stream_json, a as LoadNativeBindingOptions, b as loadNativeAnonymizeBinding, c as NativeRequire, d as availableDefaultNativePipelineLanguages, f as available_default_native_pipeline_languages, g as diagnostics_json, h as create_native_pipeline_from_default_package, i as DefaultNativePipelineWarmup, j as readNativePipelinePackageFile, k as readDefaultNativePipelinePackageFile, l as NativeSdkOptions, m as createNativePipelineFromPackageFile, n as DefaultNativePipelinePackageFileOptions, o as NativeLibc, p as createNativePipelineFromDefaultPackage, q as assertNativePipelineSupported, r as DefaultNativePipelinePackageOptions, s as NativePipelinePackageFileOptions, t as DEFAULT_NATIVE_PIPELINE_WARMUPS, u as NativeSdkPackageOptions, v as getDefaultNativePipeline, w as normalize_for_search, x as load_prepared_package, y as get_default_native_pipeline, z as redact_text_json } from "./native-node.mjs";
5
4
 
6
- //#region src/types.d.ts
7
- /**
8
- * Fields shared by every entity span in the source text.
9
- */
10
- type EntityBase = {
11
- start: number;
12
- end: number;
13
- label: string;
14
- text: string;
15
- score: number;
16
- sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
17
- };
18
- /**
19
- * A PII entity span found by a primary detection layer
20
- * (regex, NER, legal forms, deny list, ...).
21
- */
22
- type DetectedEntity = EntityBase & {
23
- source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
24
- };
25
- /**
26
- * An alias mention of a previously detected entity: a
27
- * defined term ("the Seller") or a propagated bare
28
- * mention ("Acme" after "Acme Corp.").
29
- *
30
- * `corefSourceText` is required by construction, so an
31
- * alias cannot exist without the link back to its source
32
- * entity. Placeholder numbering reads it to give the
33
- * alias the same placeholder as the source. The link
34
- * travels with the entity instead of living in a
35
- * side-channel map that a producer could forget to
36
- * write — or that a later pass could clear.
37
- */
38
- type CorefAliasEntity = EntityBase & {
39
- source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
40
- corefSourceText: string;
41
- };
42
- /**
43
- * A detected PII entity span in the source text.
44
- * Every detection layer produces these.
45
- */
46
- type Entity = DetectedEntity | CorefAliasEntity;
47
- /**
48
- * Entity after human review. Extends the base Entity
49
- * with a review decision.
50
- */
51
- type ReviewDecision = "confirmed" | "rejected" | "relabeled";
52
- type ReviewedEntity = Entity & {
53
- decision?: ReviewDecision;
54
- originalLabel?: string;
55
- };
56
- /**
57
- * A single entry in the workspace-scoped gazetteer
58
- * (deny list). Persisted in IndexedDB.
59
- */
60
- type GazetteerEntry = {
61
- id: string;
62
- canonical: string;
63
- label: string;
64
- variants: string[];
65
- workspaceId: string;
66
- createdAt: number;
67
- source: "manual" | "confirmed-from-model";
68
- };
69
- /** Extraction strategy — closed discriminated union. */
70
- type TriggerStrategy = {
71
- type: "to-next-comma";
72
- /**
73
- * Optional list of lowercase keywords that terminate
74
- * the value scan, in addition to commas/newlines. Useful
75
- * for triggers like court names that may continue past
76
- * a missing comma into adjacent clause text ("Městským
77
- * soudem v Praze dne 1. 1. 2020"); listing `"dne"` here
78
- * stops the scan at the date boundary. Matched on a
79
- * word-boundary, case-insensitive.
80
- */
81
- stopWords?: string[];
82
- /**
83
- * Hard cap on the captured span length, in characters,
84
- * regardless of where the next comma / stop char sits.
85
- * Use for triggers that label short formulaic phrases
86
- * ("State of Delaware") and must not absorb the rest
87
- * of a long forum-selection clause when the comma is
88
- * sentences away. Falls back to the default 100-char
89
- * fallback when omitted.
90
- */
91
- maxLength?: number;
92
- } | {
93
- type: "to-end-of-line";
94
- } | {
95
- type: "n-words";
96
- count: number;
97
- } | {
98
- type: "company-id-value";
99
- } | {
100
- type: "address";
101
- maxChars?: number;
102
- } | {
103
- /**
104
- * Extract the first regex match in the value text.
105
- * Useful for shape-bounded values that follow a
106
- * label on the same line as other fields, where
107
- * `to-end-of-line` would over-capture. The pattern
108
- * is anchored to the start of the (already
109
- * leading-whitespace-stripped) value, so use
110
- * `(?:.*?)` prefix only when intentional.
111
- */
112
- type: "match-pattern";
113
- pattern: string;
114
- flags?: string;
115
- };
116
- /** Validation rules — closed discriminated union. */
117
- type TriggerValidation = {
118
- type: "starts-uppercase";
119
- } | {
120
- type: "min-length";
121
- min: number;
122
- } | {
123
- type: "max-length";
124
- max: number;
125
- } | {
126
- type: "no-digits";
127
- } | {
128
- type: "has-digits";
129
- } | {
130
- type: "matches-pattern";
131
- pattern: string;
132
- flags?: string;
133
- }
134
- /**
135
- * Run a named stdnum validator (checksum + length)
136
- * against the captured value. Keeps the trigger
137
- * path symmetrical with the formatted-regex
138
- * detectors so e.g. `CPF nº 00000000000` does not
139
- * survive as a tax-ID entity.
140
- */
141
- | {
142
- type: "valid-id";
143
- validator: ValidIdValidator;
144
- };
145
- /** Built-in stdnum validators that can be referenced
146
- * by `valid-id` validations. */
147
- type ValidIdValidator = "br.cpf" | "br.cnpj" | "us.rtn";
148
- /** Auto-generated trigger variants — closed set. */
149
- type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
150
- /** V2 trigger config entry (JSON shape). */
151
- type TriggerGroupConfig = {
152
- id?: string;
153
- triggers: string[];
154
- label: string;
155
- strategy: TriggerStrategy;
156
- extensions?: TriggerExtension[];
157
- validations?: TriggerValidation[];
158
- /** When true, include the trigger text in the
159
- * entity span (e.g., court names). */
160
- includeTrigger?: boolean;
161
- };
162
- /** Compiled validation with pre-built regex. */
163
- type CompiledValidation = {
164
- type: "starts-uppercase";
165
- re: RegExp;
166
- } | {
167
- type: "min-length";
168
- min: number;
169
- } | {
170
- type: "max-length";
171
- max: number;
172
- } | {
173
- type: "no-digits";
174
- re: RegExp;
175
- } | {
176
- type: "has-digits";
177
- re: RegExp;
178
- } | {
179
- type: "matches-pattern";
180
- re: RegExp;
181
- } | {
182
- type: "valid-id";
183
- check: (value: string) => boolean;
184
- };
185
- /**
186
- * Runtime rule — one per trigger string after
187
- * expansion. Fed to the Aho-Corasick automaton.
188
- */
189
- type TriggerRule = {
190
- trigger: string;
191
- label: string;
192
- strategy: TriggerStrategy;
193
- validations: CompiledValidation[];
194
- includeTrigger: boolean;
195
- };
196
- /** Per-label operator selection. Key is the entity label. */
197
- type OperatorConfig = {
198
- /** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
199
- redactString: string;
200
- };
201
- /** Whether an operator produces a reversible redaction entry. */
202
- type OperatorReversibility = "reversible" | "irreversible";
203
- type AnonymisationOperator = {
204
- type: OperatorType;
205
- reversibility: OperatorReversibility;
206
- /**
207
- * Apply the operator to a single entity occurrence.
208
- * Returns the replacement string to embed in the document.
209
- */
210
- apply: (text: string, label: string, placeholder: string, redactString: string) => string;
211
- };
212
- /**
213
- * Redacted document output with stable entity mapping.
214
- */
215
- type RedactionResult = {
216
- redactedText: string;
217
- /**
218
- * Maps placeholder to original text. Only populated for
219
- * reversible operators (replace). Empty for redact.
220
- */
221
- redactionMap: Map<string, string>; /** Maps placeholder to the operator that produced it. */
222
- operatorMap: Map<string, OperatorType>;
223
- entityCount: number;
224
- };
225
- /**
226
- * Configuration for the detection pipeline.
227
- */
228
- type DenyListCategory = "Names" | "Places" | "Addresses" | "Courts" | "Financial" | "Government" | "Healthcare" | "Education" | "Political" | "Organizations" | "International";
229
- /**
230
- * Metadata for a single dictionary entry in the
231
- * deny-list system. Mirrors the shape from
232
- * the anonymize-data package so consumers can pass
233
- * pre-loaded data without a runtime dependency.
234
- */
235
- type DictionaryMeta = {
236
- label: string;
237
- category: DenyListCategory;
238
- country: string | null;
239
- };
240
- /**
241
- * Caller-supplied exact terms for deny-list matching.
242
- * These entries are merged with the published deny-list
243
- * dictionaries when `enableDenyList` is enabled.
244
- */
245
- type CustomDenyListEntry = {
246
- value: string;
247
- label: string;
248
- variants?: readonly string[];
249
- };
250
- /**
251
- * Caller-supplied regex detector. The pattern is passed
252
- * to the underlying text-search regex engine, so use its
253
- * supported regex syntax. Inline flags such as `(?i)` are
254
- * accepted when supported by that engine.
255
- */
256
- type CustomRegexPattern = {
257
- pattern: string;
258
- label: string;
259
- score?: number;
260
- };
261
- /**
262
- * Pre-loaded dictionary data for dependency injection.
263
- * Consumers that want name/city/deny-list detection
264
- * load dictionaries themselves (e.g. from the
265
- * anonymize-data package) and pass them here; the
266
- * anonymize package has zero cross-package imports.
267
- *
268
- * All fields are optional. When a field is absent,
269
- * the corresponding detection path is skipped (same
270
- * behavior as when no dictionaries are available).
271
- */
272
- type Dictionaries = {
273
- /**
274
- * First names per language code (e.g., "cs", "de").
275
- * Merged with legacy config names at init time.
276
- */
277
- firstNames?: Readonly<Record<string, readonly string[]>>;
278
- /**
279
- * Surnames per language code.
280
- * Merged with legacy config names at init time.
281
- */
282
- surnames?: Readonly<Record<string, readonly string[]>>;
283
- /**
284
- * Non-Western name tokens per locale code
285
- * (e.g., "in", "ar", "ja-latn", "ko", "zh-latn",
286
- * "th", "vi", "fil", "id"). Merged with bundled
287
- * names-nw-*.json data at init time.
288
- */
289
- nonWesternNames?: Readonly<Record<string, readonly string[]>>;
290
- /**
291
- * Pre-loaded deny-list dictionaries keyed by
292
- * dictionary ID (e.g., "courts/CZ", "banks/DE").
293
- * Each value is the array of terms for that
294
- * dictionary.
295
- */
296
- denyList?: Readonly<Record<string, readonly string[]>>;
297
- /**
298
- * Metadata per dictionary ID. Required when
299
- * `denyList` is provided so the pipeline knows
300
- * labels, categories, and country filters.
301
- */
302
- denyListMeta?: Readonly<Record<string, DictionaryMeta>>;
303
- /**
304
- * Pre-loaded city names, already merged across
305
- * all desired countries.
306
- *
307
- * Prefer `citiesByCountry` when callers also pass
308
- * `denyListCountries` / `denyListRegions`; merged
309
- * city arrays cannot be scoped after injection.
310
- */
311
- cities?: readonly string[];
312
- /**
313
- * Pre-loaded city names keyed by ISO 3166-1 alpha-2
314
- * country code. When provided, the deny-list builder
315
- * applies `denyListCountries` / `denyListRegions`
316
- * before adding city patterns to the search automaton.
317
- */
318
- citiesByCountry?: Readonly<Record<string, readonly string[]>>;
319
- };
320
- type PipelineConfig = {
321
- threshold: number;
322
- enableTriggerPhrases: boolean;
323
- enableRegex: boolean;
324
- /**
325
- * Enables legal-form organization detection.
326
- * Required for typed callers; legacy untyped
327
- * callers that omit this field are treated as
328
- * enabled at runtime for backward compatibility.
329
- */
330
- enableLegalForms: boolean;
331
- /**
332
- * Enables first-name/surname/title corpus matching.
333
- * When deny-list mode is enabled, this also controls
334
- * whether name-corpus entries are injected into the
335
- * deny-list search automaton.
336
- */
337
- enableNameCorpus: boolean;
338
- /**
339
- * Optional language scope for first-name/surname
340
- * dictionaries, using the keys present in
341
- * `dictionaries.firstNames` / `dictionaries.surnames`
342
- * (for example `["en", "de"]`). When omitted, all
343
- * injected name languages are used for backward
344
- * compatibility.
345
- */
346
- nameCorpusLanguages?: string[];
347
- enableDenyList: boolean;
348
- denyListCountries?: string[];
349
- denyListRegions?: string[];
350
- denyListExcludeCategories?: string[];
351
- /**
352
- * Caller-owned exact terms to match through the
353
- * deny-list layer. Requires `enableDenyList: true`.
354
- */
355
- customDenyList?: readonly CustomDenyListEntry[];
356
- /**
357
- * Caller-owned regex detectors. Requires
358
- * `enableRegex: true`.
359
- */
360
- customRegexes?: readonly CustomRegexPattern[];
361
- enableGazetteer: boolean;
362
- /**
363
- * Detect country names (ISO 3166-1 names, curated
364
- * aliases, alpha-3 codes). Defaults to true. Names
365
- * span all manifest languages plus widely-used
366
- * additions (Dutch, Russian, Chinese, Arabic, etc.).
367
- */
368
- enableCountries?: boolean;
369
- enableNer: boolean;
370
- enableConfidenceBoost: boolean;
371
- enableCoreference: boolean;
372
- enableZoneClassification?: boolean;
373
- enableHotwordRules?: boolean;
374
- /**
375
- * Requested output labels. An empty array means
376
- * "do not filter by label" for deterministic
377
- * detectors; NER falls back to DEFAULT_ENTITY_LABELS.
378
- */
379
- labels: string[];
380
- workspaceId: string;
381
- /**
382
- * Pre-loaded dictionary data for name, deny-list,
383
- * and city detection. When omitted, dictionary-based
384
- * detection paths are skipped. Consumers load from
385
- * the anonymize-data package and pass the data here.
386
- */
387
- dictionaries?: Dictionaries;
388
- };
389
- //#endregion
390
- //#region src/detectors/regex.d.ts
391
- type RegexMeta = {
392
- label: string;
393
- score: number;
394
- sourceDetail?: Entity["sourceDetail"]; /** Post-match stdnum validator for confirmation. */
395
- validator?: Validator;
396
- };
397
- /** Flat pattern array for text-search. */
398
- declare const REGEX_PATTERNS: readonly string[];
399
- /** Parallel metadata. Index = pattern index. */
400
- declare const REGEX_META: readonly RegexMeta[];
401
- /**
402
- * Get dynamically built date patterns from
403
- * date-months.json. Returns a cached promise; the JSON
404
- * is loaded only once.
405
- */
406
- declare const getDatePatterns: () => Promise<string[]>;
407
- /** Date pattern metadata (all are score 1 dates). */
408
- declare const DATE_PATTERN_META: Readonly<RegexMeta>;
409
- /**
410
- * Get dynamically built monetary amount patterns from
411
- * currencies.json. Returns a cached promise; the JSON
412
- * is loaded only once.
413
- */
414
- declare const getCurrencyPatterns: () => Promise<string[]>;
415
- /** Currency pattern metadata (score 0.9). */
416
- declare const CURRENCY_PATTERN_META: Readonly<RegexMeta>;
417
- /**
418
- * Process regex matches from the unified search.
419
- * Receives all matches; filters to the regex slice
420
- * via sliceStart/sliceEnd. Local index into META is
421
- * match.pattern - sliceStart.
422
- *
423
- * For stdnum-derived patterns (those with a validator
424
- * in META), the matched text is passed through the
425
- * validator's validate() method. If validation fails,
426
- * the match is discarded as a false positive.
427
- */
428
- declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, meta_: readonly RegexMeta[]) => Entity[];
429
- //#endregion
430
- //#region src/detectors/deny-list.d.ts
431
- type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries" | "enableCountries">;
432
- /**
433
- * Source tag for each pattern in the automaton.
434
- * "deny-list" = standard deny list entry
435
- * "city" = city dictionary entry
436
- * "custom-deny-list" = caller-owned exact term
437
- * "first-name" = name corpus first name
438
- * "surname" = name corpus surname
439
- * "title" = academic/professional title
440
- */
441
- type PatternSource = "deny-list" | "city" | "custom-deny-list" | "first-name" | "surname" | "title";
442
- type PatternLabels = string | string[];
443
- type PatternSources = PatternSource | PatternSource[];
444
- /**
445
- * Pre-built deny list data. Constructed once by
446
- * `buildDenyList`, reused across `processDenyListMatches`
447
- * calls. Contains PatternEntry[] for the unified builder
448
- * plus parallel label/source arrays for post-processing.
449
- */
450
- type DenyListData = {
451
- /**
452
- * Maps pattern index → entity labels (plural).
453
- * Same pattern can have multiple labels when it
454
- * appears in multiple dictionaries (e.g., "Denver"
455
- * is both a person name and a city name).
456
- */
457
- labels: PatternLabels[]; /** Maps pattern index → labels contributed by custom entries. */
458
- customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
459
- originals: string[]; /** Maps pattern index → source types (plural). */
460
- sources: PatternSources[];
461
- };
462
- /**
463
- * Resolve which dictionaries to load based on country
464
- * and category filters, then build the deny list data.
465
- * The returned data provides PatternEntry[] for the
466
- * unified builder and parallel arrays for
467
- * post-processing.
468
- *
469
- * Dictionary data is injected via `config.dictionaries`.
470
- * Returns null if no dictionaries are provided.
471
- */
472
- declare const buildDenyList: (config: DenyListConfig, ctx?: PipelineContext) => Promise<DenyListData | null>;
473
- /**
474
- * Ensure all deny-list support data (stopwords, allow
475
- * list, person stopwords, generic roles) is loaded on
476
- * the given context. Call this before
477
- * processDenyListMatches / filterFalsePositives when
478
- * the search instance was built on a different context
479
- * (e.g. cachedSearch).
480
- */
481
- declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Dictionaries, nameCorpusLanguages?: readonly string[]) => Promise<void>;
482
- /**
483
- * Process deny list matches from the unified search.
484
- * Receives all matches; filters to the deny list slice
485
- * via sliceStart/sliceEnd. Local index into data.labels,
486
- * data.originals, data.sources is match.pattern - sliceStart.
487
- *
488
- * Two-pass approach to reduce false positives:
489
- * 1. Collect all matches (case-insensitive,
490
- * whole-word via Rust automaton)
491
- * 2. Require uppercase start in source text
492
- * 3. For person names, require at least one
493
- * mid-sentence occurrence to prove proper noun
494
- * 4. Return all occurrences of validated terms
495
- */
496
- declare const processDenyListMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: DenyListData, ctx?: PipelineContext) => Entity[];
497
- //#endregion
498
- //#region src/detectors/countries.d.ts
499
- /**
500
- * Pre-built country patterns + parallel label/source
501
- * metadata. Constructed once and reused across pipeline
502
- * runs.
503
- */
504
- type CountryData = {
505
- /** Maps local pattern index to entity label. Always "country". */labels: string[];
506
- /**
507
- * Maps local pattern index to the alpha-2 ISO code the
508
- * pattern resolves to. Used for downstream coreference /
509
- * placeholder grouping.
510
- */
511
- isoCodes: string[]; /** Maps local pattern index to pattern variant kind. */
512
- variants: CountryVariant[];
513
- };
514
- type CountryVariant = "name" | "alias" | "alpha3" | "alpha2";
515
- //#endregion
516
- //#region src/build-unified-search.d.ts
517
- type PatternSlice = {
518
- start: number;
519
- end: number;
520
- };
521
- type GazetteerData = {
522
- /** Maps local pattern index to entry label. */labels: string[];
523
- /**
524
- * Whether each pattern is fuzzy (distance > 0).
525
- * Used by the post-processor to assign scores.
526
- */
527
- isFuzzy: boolean[];
528
- };
529
- type UnifiedSearchInstance = {
530
- /** Regex + triggers + legal-forms. */tsRegex: TextSearch; /** Caller-owned custom regexes, isolated for overlap preservation. */
531
- tsCustomRegex: TextSearch; /** Deny-list + street-types + gazetteer. */
532
- tsLiterals: TextSearch;
533
- slices: {
534
- regex: PatternSlice;
535
- customRegex: PatternSlice;
536
- legalForms: PatternSlice;
537
- triggers: PatternSlice;
538
- denyList: PatternSlice;
539
- streetTypes: PatternSlice;
540
- gazetteer: PatternSlice;
541
- countries: PatternSlice;
542
- };
543
- regexMeta: readonly RegexMeta[];
544
- customRegexMeta: readonly RegexMeta[];
545
- triggerRules: readonly TriggerRule[];
546
- denyListData: DenyListData | null;
547
- gazetteerData: GazetteerData | null;
548
- countryData: CountryData | null;
549
- };
550
- declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: GazetteerEntry[], ctx?: PipelineContext) => Promise<UnifiedSearchInstance>;
551
- //#endregion
552
- //#region src/context.d.ts
553
- /**
554
- * Build a stable cache key for an entity that survives
555
- * shallow copies (spread). Uses position + label so the
556
- * key is identical for the original object and any
557
- * `{ ...entity }` copy produced by mergeAndDedup.
558
- *
559
- * @deprecated No longer used internally: coref alias
560
- * links travel on the entities themselves
561
- * (`corefSourceText`). Kept for API compatibility.
562
- */
563
- declare const corefKey: (e: Entity) => string;
564
- /**
565
- * Compiled RegExp pattern used for coreference
566
- * definition extraction.
567
- */
568
- type DefinitionPattern = {
569
- pattern: RegExp;
570
- };
571
- /**
572
- * Cached data for the name corpus detector.
573
- * Populated by initNameCorpus; consumed by
574
- * detectNameCorpus and deny-list AC integration.
575
- */
576
- type NameCorpusData = {
577
- firstNames: ReadonlySet<string>;
578
- surnames: ReadonlySet<string>;
579
- titleTokens: ReadonlySet<string>;
580
- /** Abbreviation-style titles whose trailing dot is
581
- * part of the title, not a sentence boundary.
582
- * Contains the lowercase, dot-stripped form
583
- * (e.g., "dr", "smt", "atty"). */
584
- titleAbbreviations: ReadonlySet<string>;
585
- excludedWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
586
- nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
587
- excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
588
- firstNamesList: readonly string[];
589
- surnamesList: readonly string[];
590
- titlesList: readonly string[];
591
- excludedList: readonly string[];
592
- nonWesternNamesList: readonly string[];
593
- excludedAllCapsList: readonly string[];
594
- };
595
- /**
596
- * All cached state for a single pipeline run (or
597
- * sequence of runs sharing the same config). Replacing
598
- * module-level singletons with this object enables
599
- * concurrent pipelines with different configs and
600
- * simplifies testing.
601
- *
602
- * Each field starts null and is populated lazily on
603
- * first use by the corresponding loader function.
604
- */
605
- type PipelineContext = {
606
- search: UnifiedSearchInstance | null;
607
- searchKey: string;
608
- searchPromise: Promise<UnifiedSearchInstance> | null;
609
- nameCorpus: NameCorpusData | null;
610
- nameCorpusKey: string;
611
- nameCorpusPromise: Promise<void> | null;
612
- stopwords: ReadonlySet<string> | null;
613
- stopwordsPromise: Promise<ReadonlySet<string>> | null;
614
- allowList: ReadonlySet<string> | null;
615
- allowListPromise: Promise<ReadonlySet<string>> | null;
616
- personStopwords: ReadonlySet<string> | null;
617
- personStopwordsPromise: Promise<ReadonlySet<string>> | null;
618
- addressStopwords: ReadonlySet<string> | null;
619
- addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
620
- firstNameExclusions: ReadonlySet<string> | null;
621
- firstNameExclusionCorpusLen: number;
622
- genericRoles: ReadonlySet<string> | null;
623
- genericRolesPromise: Promise<ReadonlySet<string>> | null;
624
- corefPatterns: DefinitionPattern[] | null;
625
- corefPatternsPromise: Promise<DefinitionPattern[]> | null;
626
- corefLoadAttempted: boolean;
627
- roleStopSet: ReadonlySet<string> | null;
628
- roleStopSetPromise: Promise<ReadonlySet<string>> | null;
629
- zoneHeadingPatterns: RegExp[] | null;
630
- zoneSigningPatterns: RegExp[] | null;
631
- zoneInitPromise: Promise<void> | null;
632
- };
633
- /** Create a fresh, empty pipeline context. */
634
- declare const createPipelineContext: () => PipelineContext;
635
- //#endregion
636
- //#region src/pipeline.d.ts
637
- /** Strip leading/trailing whitespace and punctuation. */
638
- declare const sanitizeEntities: (entities: Entity[]) => Entity[];
639
- declare const mergeAndDedup: (...layers: Entity[][]) => Entity[];
640
- type NerInferenceFn = (fullText: string, labels: string[], threshold: number, signal?: AbortSignal) => Promise<Entity[]>;
641
- type PipelineSearchOptions = {
642
- config: PipelineConfig;
643
- gazetteerEntries?: GazetteerEntry[];
644
- context?: PipelineContext;
645
- };
646
- /**
647
- * Pre-build and cache the unified search instance for a
648
- * pipeline configuration. Use the same context in
649
- * `runPipeline` to reuse the prepared automata without
650
- * passing `cachedSearch` around manually.
651
- */
652
- declare const preparePipelineSearch: ({
653
- config,
654
- gazetteerEntries,
655
- context
656
- }: PipelineSearchOptions) => Promise<UnifiedSearchInstance>;
657
- /**
658
- * Options for {@link runPipeline}.
659
- *
660
- * @property cachedSearch Pre-built search instance.
661
- * When provided, `config` and `gazetteerEntries`
662
- * are not used for building; the caller must
663
- * ensure the instance matches both parameters.
664
- */
665
- type PipelineOptions = {
666
- fullText: string;
667
- config: PipelineConfig;
668
- gazetteerEntries: GazetteerEntry[];
669
- nerInference?: NerInferenceFn | null;
670
- onProgress?: (step: string, detail: string) => void;
671
- cachedSearch?: UnifiedSearchInstance;
672
- signal?: AbortSignal;
673
- context?: PipelineContext;
674
- };
675
- /**
676
- * Run the full detection pipeline.
677
- *
678
- * Two TextSearch instances scan the text (regex +
679
- * literals). Results are dispatched to each
680
- * detector's post-processor by pattern index range.
681
- *
682
- * Pass an AbortSignal to cancel the pipeline between
683
- * stages. Throws a DOMException with name "AbortError"
684
- * when cancelled.
685
- *
686
- * Pass an optional `context` to isolate cached state
687
- * from other pipeline runs. If omitted, a module-level
688
- * default context is used (backward compatible).
689
- */
690
- declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
691
- //#endregion
692
5
  //#region src/redact.d.ts
693
- /**
694
- * Build a stable mapping from entity text to numbered
695
- * placeholders. Same real-world value always maps to the
696
- * same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
697
- * both become [PERSON_1]).
698
- *
699
- * Placeholder format: [LABEL_N] where LABEL is uppercase
700
- * and N is a 1-based counter per label.
701
- *
702
- * @param _ctx Unused. Kept for signature compatibility;
703
- * coref alias links now travel on the entities
704
- * themselves (`corefSourceText`).
705
- */
706
- declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext) => Map<string, string>;
707
- /**
708
- * Apply redactions to the source text, replacing each
709
- * confirmed entity span using the configured operator.
710
- *
711
- * Co-references are consistent: if the same text appears
712
- * multiple times, all occurrences get the same placeholder.
713
- *
714
- * @param ctx Pipeline context. Must be the same instance
715
- * passed to `runPipeline` (or `findCoreferenceSpans`)
716
- * so coreference placeholder links are preserved.
717
- * Defaults to `defaultContext` for single-tenant usage.
718
- */
719
- declare const redactText: (fullText: string, entities: Entity[], config?: OperatorConfig, ctx?: PipelineContext) => RedactionResult;
720
6
  /**
721
7
  * Serialize the redaction key to JSON for export.
722
8
  * Includes operator metadata so the export is self-describing.
@@ -729,455 +15,5 @@ declare const exportRedactionKey: (redactionMap: Map<string, string>, operatorMa
729
15
  */
730
16
  declare const deanonymise: (redactedText: string, redactionMap: Map<string, string>) => string;
731
17
  //#endregion
732
- //#region src/operators.d.ts
733
- declare const OPERATOR_REGISTRY: {
734
- readonly replace: AnonymisationOperator;
735
- readonly redact: AnonymisationOperator;
736
- };
737
- /**
738
- * Default operator config: replace for all labels.
739
- * Preserves existing pipeline behaviour.
740
- */
741
- declare const DEFAULT_OPERATOR_CONFIG: OperatorConfig;
742
- /**
743
- * Resolve the operator for a label, falling back to "replace".
744
- */
745
- declare const resolveOperator: (config: OperatorConfig, label: string) => OperatorType;
746
- //#endregion
747
- //#region src/detectors/legal-forms.d.ts
748
- declare const warmLegalRoleHeads: () => Promise<void>;
749
- /**
750
- * Build legal form regex pattern strings.
751
- * Returns an array of regex strings for the unified
752
- * TextSearch builder. Empty if data package is not
753
- * installed.
754
- */
755
- declare const buildLegalFormPatterns: () => Promise<string[]>;
756
- /**
757
- * Process legal form matches from the unified search.
758
- * Receives all matches; filters to the legal forms
759
- * slice via sliceStart/sliceEnd.
760
- *
761
- * The role-head trimming step reads per-language data from
762
- * a cache that `runPipeline` warms via `warmLegalRoleHeads()`
763
- * before calling this. Callers that invoke
764
- * `processLegalFormMatches` directly (without going through
765
- * `runPipeline`) must `await warmLegalRoleHeads()` first;
766
- * otherwise the trim falls back to a no-op and sentence-
767
- * fragment fixes do not apply.
768
- */
769
- declare const processLegalFormMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText?: string, options?: {
770
- suppressExtendBackward?: boolean;
771
- }) => Entity[];
772
- //#endregion
773
- //#region src/detectors/triggers.d.ts
774
- type TriggerPatterns = {
775
- patterns: string[];
776
- rules: TriggerRule[];
777
- };
778
- declare const buildTriggerPatterns: () => Promise<TriggerPatterns>;
779
- /**
780
- * Process trigger matches from the unified search.
781
- * Receives all matches; filters to the trigger slice
782
- * via sliceStart/sliceEnd. Uses fullText for value
783
- * extraction (the unified search runs on lowercased
784
- * text, but extraction needs original casing).
785
- */
786
- declare const processTriggerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, rules: readonly TriggerRule[]) => Entity[];
787
- //#endregion
788
- //#region src/detectors/address-seeds.d.ts
789
- declare const buildStreetTypePatterns: () => Promise<string[]>;
790
- /**
791
- * Process address seeds from the unified search.
792
- * Receives all matches; filters to the street types
793
- * slice via sliceStart/sliceEnd. Uses fullText and
794
- * existingEntities for seed collection, clustering,
795
- * expansion, and scoring.
796
- *
797
- * Runs as a post-processor after all other detectors,
798
- * using their output as seed sources.
799
- */
800
- declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
801
- //#endregion
802
- //#region src/detectors/gazetteer.d.ts
803
- /**
804
- * Build TextSearch-compatible patterns from gazetteer
805
- * entries. Returns:
806
- * - Exact literal patterns for all terms
807
- * - Fuzzy patterns (distance: 2) for terms >= 4 chars
808
- * - Parallel metadata arrays for post-processing
809
- *
810
- * Patterns are ordered: all exact first, then all
811
- * fuzzy. The isFuzzy array marks which are which.
812
- */
813
- declare const buildGazetteerPatterns: (entries: GazetteerEntry[]) => {
814
- patterns: PatternEntry[];
815
- data: GazetteerData;
816
- };
817
- /**
818
- * Process gazetteer matches from the unified literal
819
- * search. Receives all matches; filters to the
820
- * gazetteer slice via sliceStart/sliceEnd.
821
- *
822
- * Exact matches get score 0.9; fuzzy matches get
823
- * 0.85. Fuzzy matches that overlap an exact match
824
- * are dropped.
825
- *
826
- * For exact matches, attempts prefix extension for
827
- * legal suffixes ("a.s.", "GmbH", "s.r.o." after
828
- * the matched term).
829
- */
830
- declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: GazetteerData) => Entity[];
831
- //#endregion
832
- //#region src/detectors/coreference.d.ts
833
- type DefinedTerm = {
834
- alias: string;
835
- label: string; /** Position of the definition in the source text */
836
- definitionStart: number; /** Original entity text the alias refers to */
837
- sourceText: string;
838
- };
839
- declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext) => Promise<DefinedTerm[]>;
840
- /**
841
- * Find all occurrences of defined-term aliases in the
842
- * full text. Returns Entity spans for each match.
843
- *
844
- * Respects word boundaries: "Kupující" must not match
845
- * inside "Kupujícímu". A match is valid only if the
846
- * character before the start and after the end are NOT
847
- * word characters (letter/digit).
848
- *
849
- * Each returned alias carries `corefSourceText` linking
850
- * it to its source entity text, for consistent
851
- * placeholder numbering.
852
- *
853
- * @param _ctx Unused. Kept for signature compatibility;
854
- * alias links now travel on the entities themselves.
855
- */
856
- declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ctx?: PipelineContext) => Entity[];
857
- //#endregion
858
- //#region src/detectors/org-propagation.d.ts
859
- /**
860
- * After the main detection pass, collect organization
861
- * entities with a legal form suffix, strip the suffix
862
- * to get the base name, and re-scan the full text for
863
- * bare mentions of that base name. Returns new entities
864
- * for occurrences not already covered.
865
- *
866
- * Propagated mentions are coref aliases: each carries
867
- * `corefSourceText` linking it to the full seed entity
868
- * text, so placeholder numbering assigns the bare
869
- * mention the same placeholder as its source ("Acme"
870
- * and "Acme Corp." both become [ORGANIZATION_1]).
871
- */
872
- declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
873
- //#endregion
874
- //#region src/detectors/names.d.ts
875
- declare const getNameCorpusNonWesternNames: (ctx?: PipelineContext) => readonly string[];
876
- /**
877
- * Load name corpus data from injected dictionaries
878
- * and legacy config files. Merges all sources.
879
- *
880
- * Safe to call multiple times; only loads once per
881
- * context. Must be called before detectNameCorpus or
882
- * the getNameCorpus*() accessors are used.
883
- *
884
- * @param dictionaries Optional pre-loaded dictionaries
885
- * with per-language first names and surnames. When
886
- * omitted, only legacy config files are used.
887
- */
888
- declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionaries, languages?: readonly string[]) => Promise<void>;
889
- type NameCorpusDetectionOptions = {
890
- mode?: "full" | "supplemental";
891
- };
892
- /**
893
- * Detect person names by looking up tokens against the
894
- * name corpus, then chaining adjacent name-like tokens.
895
- * Handles both Western and non-Western name patterns.
896
- *
897
- * Requires initNameCorpus() to have been called first.
898
- * If not initialized, returns an empty array.
899
- *
900
- * Scoring (Western):
901
- * TITLE + NAME/SURNAME → 0.95
902
- * NAME + NAME/SURNAME → 0.9
903
- * SURNAME + NAME/SURNAME → 0.9
904
- * NAME + CAPITALIZED → 0.7
905
- * ABBREVIATION + NAME → 0.7
906
- * Standalone NAME → 0.5 (low confidence)
907
- * Standalone SURNAME → skip (too ambiguous)
908
- *
909
- * Scoring (non-Western, when chain contains nonWestern tokens):
910
- * TITLE + (nonWestern|CAPITALIZED) → 0.95
911
- * JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
912
- * ARABIC_CONNECTOR + nonWestern → 0.9
913
- * 2+ nonWestern tokens → 0.9
914
- * nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
915
- * Standalone nonWestern mid-sentence → 0.5
916
- */
917
- declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext, options?: NameCorpusDetectionOptions) => Entity[];
918
- //#endregion
919
- //#region src/unified-search.d.ts
920
- type UnifiedResult = {
921
- /** All matches from both instances combined. */regexMatches: Match[];
922
- customRegexMatches: Match[];
923
- literalMatches: Match[];
924
- };
925
- declare const runUnifiedSearch: (instance: UnifiedSearchInstance, fullText: string) => UnifiedResult;
926
- //#endregion
927
- //#region src/regions.d.ts
928
- /**
929
- * Geographic regions and country code mappings for
930
- * scoping deny list dictionaries.
931
- */
932
- declare const REGIONS: {
933
- readonly Global: null;
934
- readonly International: null;
935
- readonly Europe: readonly ["AL", "AD", "AT", "BE", "BA", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IS", "IE", "IT", "XK", "LV", "LI", "LT", "LU", "MD", "ME", "MK", "MT", "MC", "NL", "NO", "PL", "PT", "RO", "RS", "SK", "SI", "ES", "SE", "CH", "UA", "GB"];
936
- readonly Americas: readonly ["US", "CA", "MX", "BR", "AR", "CL", "CO", "PE", "EC", "VE", "UY", "PY", "BO", "CR", "PA", "DO", "GT", "HN", "SV", "NI", "CU"];
937
- readonly AsiaPacific: readonly ["AU", "NZ", "JP", "KR", "CN", "TW", "SG", "MY", "TH", "VN", "PH", "ID", "IN", "PK", "BD", "LK", "NP", "HK", "MO"];
938
- readonly MENA: readonly ["AE", "SA", "IL", "TR", "EG", "JO", "LB", "IQ", "IR", "QA", "KW", "BH", "OM", "MA", "TN", "DZ", "LY", "SY", "YE", "PS"];
939
- readonly SubSaharanAfrica: readonly ["ZA", "NG", "KE", "GH", "TZ", "ET", "SN", "CI", "CM", "UG", "RW", "MZ", "AO", "ZW", "BW", "NA", "MU"];
940
- readonly EU: readonly ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE"];
941
- readonly DACH: readonly ["DE", "AT", "CH"];
942
- readonly Nordics: readonly ["DK", "SE", "NO", "FI", "IS"];
943
- readonly CEE: readonly ["CZ", "SK", "PL", "HU", "RO", "BG", "HR", "SI", "LT", "LV", "EE"];
944
- readonly Anglosphere: readonly ["GB", "US", "CA", "AU", "NZ", "IE"];
945
- readonly Benelux: readonly ["BE", "NL", "LU"];
946
- readonly GulfStates: readonly ["AE", "SA", "QA", "KW", "BH", "OM"];
947
- readonly SouthAsia: readonly ["IN", "PK", "BD", "LK", "NP"];
948
- readonly EastAsia: readonly ["CN", "JP", "KR", "TW"];
949
- readonly SoutheastAsia: readonly ["SG", "MY", "TH", "VN", "PH", "ID"];
950
- readonly Oceania: readonly ["AU", "NZ"];
951
- };
952
- type RegionId = keyof typeof REGIONS;
953
- type RegionArrays = { [K in RegionId]: (typeof REGIONS)[K] };
954
- type NonNullRegion = { [K in RegionId as RegionArrays[K] extends null ? never : K]: RegionArrays[K] };
955
- type CountryCode = NonNullRegion[keyof NonNullRegion][number];
956
- /**
957
- * Expand region names to country codes and merge with
958
- * explicit country codes. Returns null when both inputs
959
- * are empty/undefined (meaning "match all countries").
960
- */
961
- declare const resolveCountries: (regions?: string[], countries?: string[]) => Set<string> | null;
962
- //#endregion
963
- //#region src/filters/false-positives.d.ts
964
- /** Ensure street-type vocabulary is loaded. */
965
- declare const initAddressComponents: () => Promise<void>;
966
- /**
967
- * Filter out entities that are likely false positives:
968
- * template placeholders, clause/section numbers,
969
- * standalone years, and generic legal role terms.
970
- *
971
- * Runs as a post-processing step after all detection
972
- * layers have merged.
973
- */
974
- declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext, fullText?: string) => Entity[];
975
- //#endregion
976
- //#region src/filters/confidence-boost.d.ts
977
- /**
978
- * Boost confidence of near-miss NER entities that appear
979
- * near high-confidence detections (regex, trigger phrase).
980
- *
981
- * If an NER entity scored between (threshold - 0.15) and
982
- * threshold, count how many confirmed entities exist within
983
- * a 150-char window. Add +0.05 per co-located entity.
984
- * If the boosted score crosses the threshold, include it.
985
- *
986
- * Only mutates score on near-miss entities; high-confidence
987
- * entities pass through unchanged.
988
- */
989
- declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
990
- //#endregion
991
- //#region src/filters/hotword-rules.d.ts
992
- type HotwordRule = {
993
- hotwords: string[];
994
- targetLabels: string[];
995
- scoreAdjustment: number;
996
- reclassifyTo?: string;
997
- proximityBefore: number;
998
- proximityAfter: number;
999
- };
1000
- /**
1001
- * Load hotword rules from the data package.
1002
- * Safe to call multiple times; subsequent calls
1003
- * are no-ops.
1004
- */
1005
- declare const initHotwordRules: () => Promise<void>;
1006
- /**
1007
- * Apply hotword context rules to detected entities.
1008
- *
1009
- * Scans `fullText` once with a single AC automaton
1010
- * for all hotwords across all rules, then checks
1011
- * proximity to each entity. Distance-decayed
1012
- * adjustment: closer hotwords give a stronger boost.
1013
- *
1014
- * Returns a new array; input entities are not mutated.
1015
- */
1016
- declare const applyHotwordRules: (entities: Entity[], fullText: string) => Entity[];
1017
- //#endregion
1018
- //#region src/filters/zone-classifier.d.ts
1019
- type DocumentZone = "header" | "signature" | "body" | "table";
1020
- type ZoneSpan = {
1021
- zone: DocumentZone;
1022
- start: number;
1023
- end: number;
1024
- };
1025
- /**
1026
- * Additive score adjustments per document zone.
1027
- * Header and signature blocks are dense with PII;
1028
- * tables often contain structured identifying data.
1029
- */
1030
- declare const ZONE_SCORE_ADJUSTMENTS: {
1031
- readonly header: 0.1;
1032
- readonly signature: 0.15;
1033
- readonly body: 0;
1034
- readonly table: 0.05;
1035
- };
1036
- /**
1037
- * Ensure config data is loaded. Call once before
1038
- * classifyZones. Safe to call multiple times.
1039
- */
1040
- declare const initZoneClassifier: (ctx?: PipelineContext) => Promise<void>;
1041
- /**
1042
- * Classify a document into zones based on
1043
- * structural heuristics. Zones are non-overlapping
1044
- * and cover the entire text.
1045
- *
1046
- * Must call `initZoneClassifier()` first.
1047
- */
1048
- declare const classifyZones: (fullText: string, ctx?: PipelineContext) => ZoneSpan[];
1049
- /**
1050
- * Apply zone-based score adjustments to entities.
1051
- * Entities in header/signature/table zones get a
1052
- * small additive boost reflecting the higher PII
1053
- * density in those regions.
1054
- *
1055
- * Returns a new array; does not mutate inputs.
1056
- */
1057
- declare const applyZoneAdjustments: (entities: Entity[], zones: ZoneSpan[]) => Entity[];
1058
- //#endregion
1059
- //#region src/gliner/types.d.ts
1060
- /**
1061
- * GLiNER inference types.
1062
- *
1063
- * Forked from gliner@0.0.19 (MIT), stripped to runtime-
1064
- * agnostic core. Original: github.com/Ingvarstep/GLiNER.js
1065
- */
1066
- type EntityResult = {
1067
- spanText: string;
1068
- start: number;
1069
- end: number;
1070
- label: string;
1071
- score: number;
1072
- };
1073
- /**
1074
- * Raw inference output: per-batch array of
1075
- * [spanText, start, end, label, score] tuples.
1076
- */
1077
- type RawInferenceResult = [string, number, number, string, number][][];
1078
- //#endregion
1079
- //#region src/gliner/decoder.d.ts
1080
- /**
1081
- * Decode span-level model logits into entity results.
1082
- */
1083
- declare const decodeSpans: (batchSize: number, inputLength: number, maxWidth: number, numEntities: number, texts: string[], batchIds: number[], batchWordsStartIdx: number[][], batchWordsEndIdx: number[][], idToClass: Record<number, string>, modelOutput: ArrayLike<number>, flatNer: boolean, threshold: number, multiLabel: boolean) => RawInferenceResult;
1084
- //#endregion
1085
- //#region src/gliner/token-decoder.d.ts
1086
- /**
1087
- * Decode token-level BIO logits into entity spans.
1088
- *
1089
- * For each word, checks if the B(egin) logit for any class
1090
- * exceeds the threshold. If so, extends the span by consuming
1091
- * subsequent I(nside) tokens of the same class.
1092
- */
1093
- declare const decodeTokenSpans: (batchSize: number, numWords: number, numEntities: number, texts: string[], batchIds: number[], batchWordsStartIdx: number[][], batchWordsEndIdx: number[][], idToClass: Record<number, string>, modelOutput: ArrayLike<number>, threshold: number) => RawInferenceResult;
1094
- //#endregion
1095
- //#region src/gliner/processor.d.ts
1096
- /** Tokenize text into words with character offsets. */
1097
- declare const tokenizeText: (text: string) => [words: string[], starts: number[], ends: number[]];
1098
- /** Prepare a complete batch for ONNX inference. */
1099
- declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: string[], maxWidth: number) => {
1100
- inputsIds: number[][];
1101
- attentionMasks: number[][];
1102
- wordsMasks: number[][];
1103
- textLengths: number[];
1104
- spanIdxs: number[][][];
1105
- spanMasks: boolean[][];
1106
- idToClass: Record<number, string>;
1107
- batchTokens: string[][];
1108
- batchWordsStartIdx: number[][];
1109
- batchWordsEndIdx: number[][];
1110
- };
1111
- //#endregion
1112
- //#region src/util/chunker.d.ts
1113
- /**
1114
- * Split text into overlapping chunks for GLiNER's
1115
- * ~512 token context window. Character-based splitting
1116
- * (rough approximation of token limits).
1117
- *
1118
- * Tries to break at sentence boundaries when possible.
1119
- */
1120
- declare const chunkText: (text: string) => string[];
1121
- /**
1122
- * Compute the byte offset of each chunk within the
1123
- * original document text.
1124
- */
1125
- declare const computeChunkOffsets: (fullText: string, chunks: string[]) => number[];
1126
- /**
1127
- * Merge entities from overlapping chunks back to
1128
- * document-level offsets. Deduplicates entities that
1129
- * appear in overlap regions (keeps highest score).
1130
- *
1131
- * Dedup invariant: each incoming entity is compared
1132
- * against the highest-scored same-label near-dup in
1133
- * its proximity window. If it loses, it is dropped.
1134
- * This does NOT guarantee that all pairwise near-dup
1135
- * relationships in the output are resolved; a lower-
1136
- * scored entity can survive if the bridging entity
1137
- * that would have replaced it was itself dropped by
1138
- * a higher-scored match.
1139
- *
1140
- * Uses a reverse-scan over the sorted merged array
1141
- * so each entity only compares against nearby
1142
- * predecessors — O(n * w) average where w is the max
1143
- * entities per POSITION_THRESHOLD window, O(n²) worst
1144
- * case when replacements dominate (splice is O(n)).
1145
- */
1146
- declare const mergeChunkEntities: (chunkOffsets: number[], chunkResults: Entity[][]) => Entity[];
1147
- //#endregion
1148
- //#region src/util/levenshtein.d.ts
1149
- /**
1150
- * Compute the Levenshtein edit distance between two
1151
- * strings. O(n*m) time, O(min(n,m)) space using a
1152
- * single-row DP approach.
1153
- */
1154
- declare const levenshtein: (rawA: string, rawB: string) => number;
1155
- //#endregion
1156
- //#region src/util/normalize.d.ts
1157
- /**
1158
- * Normalize typographic variants for search matching.
1159
- *
1160
- * Legal documents (especially Czech/German) use
1161
- * non-breaking spaces, smart quotes, and en/em dashes
1162
- * that differ from their ASCII equivalents. Since all
1163
- * replacements are same-length (single code unit →
1164
- * single code unit), character offsets remain valid.
1165
- *
1166
- * Lives here (application layer) rather than in the
1167
- * AC library: what to normalize is domain-specific.
1168
- *
1169
- * Uses a char-code lookup (`Map<number, number>`) and
1170
- * `Uint16Array` instead of 7 sequential `replaceAll`
1171
- * calls. For a 50 KB document this eliminates ~350 KB
1172
- * of intermediate string allocations.
1173
- *
1174
- * When no replaceable characters are present (common
1175
- * for plain-text inputs), a fast-path scan returns the
1176
- * original string without any allocation. When special
1177
- * characters exist, the string is scanned twice: once
1178
- * to detect, once to build the replacement array.
1179
- */
1180
- declare const normalizeForSearch: (text: string) => string;
1181
- //#endregion
1182
- export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
18
+ export { type AnonymisationOperator, type CustomDenyListEntry, type CustomRegexPattern, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DETECTION_SOURCES, DETECTOR_PRIORITY, DefaultNativePipelinePackageFileOptions, DefaultNativePipelinePackageOptions, DefaultNativePipelineWarmup, type DenyListCategory, type DetectionSource, type Dictionaries, type DictionaryMeta, type Entity, type GazetteerEntry, LoadNativeBindingOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeDiagnosticsBatchCallback, NativeLibc, NativeNormalizeOptions, NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, NativePipelineEntity, NativePipelineFromPackageOptions, NativePipelinePackageFileOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, NativePreparedSearchBinding, NativeRedactionResult, NativeRequire, NativeResultEventCallback, NativeSdkOptions, NativeSdkPackageOptions, NativeSearchPackageInput, NativeSearchPackageOptions, NativeStaticRedactionResult, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, type RedactionResult, type ReviewDecision, type ReviewedEntity, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, deanonymise, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
1183
19
  //# sourceMappingURL=index.d.mts.map