@stll/anonymize-wasm 1.5.0 → 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -2
- package/dist/address-boundaries.mjs +2 -0
- package/dist/address-jurisdiction-prefixes.mjs +16 -0
- package/dist/address-jurisdiction-prefixes.mjs.map +1 -0
- package/dist/address-stop-keywords.mjs +12 -1
- package/dist/address-unit-abbreviations.mjs +15 -0
- package/dist/address-unit-abbreviations.mjs.map +1 -0
- package/dist/clause-noun-heads.mjs +5 -1
- package/dist/coreference-org-determiners.mjs +19 -0
- package/dist/coreference-org-determiners.mjs.map +1 -0
- package/dist/defined-term-heads.mjs +15 -0
- package/dist/defined-term-heads.mjs.map +1 -0
- package/dist/false-positive-shapes.mjs +36 -0
- package/dist/false-positive-shapes.mjs.map +1 -0
- package/dist/legal-role-heads.cs.mjs +6 -0
- package/dist/organization-unit-heads.mjs +20 -0
- package/dist/organization-unit-heads.mjs.map +1 -0
- package/dist/person-stopwords.mjs +205 -199
- package/dist/signing-clauses.mjs +33 -9
- package/dist/wasm.d.mts +665 -39
- package/dist/wasm.mjs +4048 -1453
- package/dist/wasm.mjs.map +1 -1
- package/package.json +2 -2
- package/dist/address-prepositions.mjs +0 -182
- package/dist/address-prepositions.mjs.map +0 -1
package/dist/wasm.d.mts
CHANGED
|
@@ -180,6 +180,7 @@ type CompiledValidation = {
|
|
|
180
180
|
re: RegExp;
|
|
181
181
|
} | {
|
|
182
182
|
type: "valid-id";
|
|
183
|
+
validator: ValidIdValidator;
|
|
183
184
|
check: (value: string) => boolean;
|
|
184
185
|
};
|
|
185
186
|
/**
|
|
@@ -257,6 +258,7 @@ type CustomRegexPattern = {
|
|
|
257
258
|
pattern: string;
|
|
258
259
|
label: string;
|
|
259
260
|
score?: number;
|
|
261
|
+
preparedArtifactPolicy?: "include" | "omit";
|
|
260
262
|
};
|
|
261
263
|
/**
|
|
262
264
|
* Pre-loaded dictionary data for dependency injection.
|
|
@@ -321,6 +323,18 @@ type PipelineConfig = {
|
|
|
321
323
|
threshold: number;
|
|
322
324
|
enableTriggerPhrases: boolean;
|
|
323
325
|
enableRegex: boolean;
|
|
326
|
+
/**
|
|
327
|
+
* Expected content language codes. When present, these
|
|
328
|
+
* derive default dictionary scopes for name corpus and
|
|
329
|
+
* deny-list matching unless the lower-level scope fields
|
|
330
|
+
* below are set explicitly.
|
|
331
|
+
*/
|
|
332
|
+
languages?: string[];
|
|
333
|
+
/**
|
|
334
|
+
* Convenience form for single-language documents. Ignored
|
|
335
|
+
* when `languages` is also provided.
|
|
336
|
+
*/
|
|
337
|
+
language?: string;
|
|
324
338
|
/**
|
|
325
339
|
* Enables legal-form organization detection.
|
|
326
340
|
* Required for typed callers; legacy untyped
|
|
@@ -391,22 +405,48 @@ type PipelineConfig = {
|
|
|
391
405
|
type RegexMeta = {
|
|
392
406
|
label: string;
|
|
393
407
|
score: number;
|
|
394
|
-
sourceDetail?: Entity["sourceDetail"];
|
|
395
|
-
|
|
408
|
+
sourceDetail?: Entity["sourceDetail"];
|
|
409
|
+
minByteLength?: number; /** Post-match stdnum validator for confirmation. */
|
|
410
|
+
validator?: Validator;
|
|
411
|
+
validatorId?: string; /** Extract the identifier portion when context is part of the regex span. */
|
|
396
412
|
validatorInput?: (text: string) => string;
|
|
413
|
+
validatorInputKind?: "digits-only" | "crypto-wallet-candidate";
|
|
397
414
|
};
|
|
398
415
|
/** Flat pattern array for text-search. */
|
|
399
416
|
declare const REGEX_PATTERNS: readonly string[];
|
|
400
417
|
/** Parallel metadata. Index = pattern index. */
|
|
401
418
|
declare const REGEX_META: readonly RegexMeta[];
|
|
419
|
+
type DateMonthData = Record<string, string[]>;
|
|
420
|
+
type YearWordData = Record<string, string[]>;
|
|
402
421
|
/**
|
|
403
422
|
* Get dynamically built date patterns from
|
|
404
423
|
* date-months.json. Returns a cached promise; the JSON
|
|
405
424
|
* is loaded only once.
|
|
406
425
|
*/
|
|
407
|
-
declare const getDatePatterns: () => Promise<string[]>;
|
|
426
|
+
declare const getDatePatterns: (languages?: readonly string[]) => Promise<string[]>;
|
|
408
427
|
/** Date pattern metadata (all are score 1 dates). */
|
|
409
428
|
declare const DATE_PATTERN_META: Readonly<RegexMeta>;
|
|
429
|
+
type MonetaryData = {
|
|
430
|
+
currencies: {
|
|
431
|
+
codes: string[];
|
|
432
|
+
symbols: string[];
|
|
433
|
+
local_names: string[];
|
|
434
|
+
};
|
|
435
|
+
amount_words: {
|
|
436
|
+
written_amount_patterns: Array<{
|
|
437
|
+
keywords: string[];
|
|
438
|
+
}>;
|
|
439
|
+
magnitude_suffixes: Array<{
|
|
440
|
+
words: string[];
|
|
441
|
+
abbreviations_case_insensitive: string[];
|
|
442
|
+
abbreviations_case_sensitive: string[];
|
|
443
|
+
}>;
|
|
444
|
+
share_quantity_terms: Array<{
|
|
445
|
+
modifiers: string[];
|
|
446
|
+
nouns: string[];
|
|
447
|
+
}>;
|
|
448
|
+
};
|
|
449
|
+
};
|
|
410
450
|
/**
|
|
411
451
|
* Get dynamically built monetary amount patterns from
|
|
412
452
|
* currencies.json. Returns a cached promise; the JSON
|
|
@@ -430,6 +470,30 @@ declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sli
|
|
|
430
470
|
//#endregion
|
|
431
471
|
//#region src/detectors/deny-list.d.ts
|
|
432
472
|
type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries" | "enableCountries">;
|
|
473
|
+
type DenyListFilterData = {
|
|
474
|
+
stopwords: string[];
|
|
475
|
+
allowList: string[];
|
|
476
|
+
personStopwords: string[];
|
|
477
|
+
personTrailingNouns: string[];
|
|
478
|
+
addressStopwords: string[];
|
|
479
|
+
addressJurisdictionPrefixes: string[];
|
|
480
|
+
streetTypes: string[];
|
|
481
|
+
addressComponentTerms: string[];
|
|
482
|
+
ambiguousStreetTypeTerms: string[];
|
|
483
|
+
firstNames: string[];
|
|
484
|
+
genericRoles: string[];
|
|
485
|
+
numberAbbrevPrefixes: string[];
|
|
486
|
+
sentenceStarters: string[];
|
|
487
|
+
trailingAddressWordExclusions: string[];
|
|
488
|
+
documentHeadingWords: string[];
|
|
489
|
+
documentHeadingOrdinalMarkers: string[];
|
|
490
|
+
definedTermCues: string[];
|
|
491
|
+
signingPlaceGuards: DenyListSigningPlaceGuardData[];
|
|
492
|
+
};
|
|
493
|
+
type DenyListSigningPlaceGuardData = {
|
|
494
|
+
prefixPhrases: string[];
|
|
495
|
+
suffixPhrases: string[];
|
|
496
|
+
};
|
|
433
497
|
/**
|
|
434
498
|
* Source tag for each pattern in the automaton.
|
|
435
499
|
* "deny-list" = standard deny list entry
|
|
@@ -459,6 +523,7 @@ type DenyListData = {
|
|
|
459
523
|
customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
|
|
460
524
|
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
461
525
|
sources: PatternSources[];
|
|
526
|
+
filters: DenyListFilterData;
|
|
462
527
|
};
|
|
463
528
|
/**
|
|
464
529
|
* Resolve which dictionaries to load based on country
|
|
@@ -496,6 +561,25 @@ declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Diction
|
|
|
496
561
|
*/
|
|
497
562
|
declare const processDenyListMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: DenyListData, ctx?: PipelineContext) => Entity[];
|
|
498
563
|
//#endregion
|
|
564
|
+
//#region src/detectors/address-seeds.d.ts
|
|
565
|
+
type AddressSeedData = {
|
|
566
|
+
boundary_words: string[];
|
|
567
|
+
br_cep_cue_words: string[];
|
|
568
|
+
unit_abbreviations: string[];
|
|
569
|
+
};
|
|
570
|
+
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
571
|
+
/**
|
|
572
|
+
* Process address seeds from the unified search.
|
|
573
|
+
* Receives all matches; filters to the street types
|
|
574
|
+
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
575
|
+
* existingEntities for seed collection, clustering,
|
|
576
|
+
* expansion, and scoring.
|
|
577
|
+
*
|
|
578
|
+
* Runs as a post-processor after all other detectors,
|
|
579
|
+
* using their output as seed sources.
|
|
580
|
+
*/
|
|
581
|
+
declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
|
|
582
|
+
//#endregion
|
|
499
583
|
//#region src/detectors/countries.d.ts
|
|
500
584
|
/**
|
|
501
585
|
* Pre-built country patterns + parallel label/source
|
|
@@ -514,11 +598,289 @@ type CountryData = {
|
|
|
514
598
|
};
|
|
515
599
|
type CountryVariant = "name" | "alias" | "alpha3" | "alpha2";
|
|
516
600
|
//#endregion
|
|
601
|
+
//#region src/filters/confidence-boost.d.ts
|
|
602
|
+
/**
|
|
603
|
+
* Boost confidence of near-miss NER entities that appear
|
|
604
|
+
* near high-confidence detections (regex, trigger phrase).
|
|
605
|
+
*
|
|
606
|
+
* If an NER entity scored between (threshold - 0.15) and
|
|
607
|
+
* threshold, count how many confirmed entities exist within
|
|
608
|
+
* a 150-char window. Add +0.05 per co-located entity.
|
|
609
|
+
* If the boosted score crosses the threshold, include it.
|
|
610
|
+
*
|
|
611
|
+
* Only mutates score on near-miss entities; high-confidence
|
|
612
|
+
* entities pass through unchanged.
|
|
613
|
+
*/
|
|
614
|
+
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
615
|
+
type AddressContextData = {
|
|
616
|
+
address_prepositions: string[];
|
|
617
|
+
temporal_prepositions: string[];
|
|
618
|
+
street_abbreviations: string[];
|
|
619
|
+
bare_house_stopwords: string[];
|
|
620
|
+
};
|
|
621
|
+
//#endregion
|
|
517
622
|
//#region src/build-unified-search.d.ts
|
|
518
623
|
type PatternSlice = {
|
|
519
624
|
start: number;
|
|
520
625
|
end: number;
|
|
521
626
|
};
|
|
627
|
+
type NativeSearchPatternKind = "literal" | "literal-with-options" | "regex" | "fuzzy";
|
|
628
|
+
type NativeSearchPattern = {
|
|
629
|
+
kind: NativeSearchPatternKind;
|
|
630
|
+
pattern: string;
|
|
631
|
+
distance?: number;
|
|
632
|
+
case_insensitive?: boolean;
|
|
633
|
+
whole_words?: boolean;
|
|
634
|
+
lazy?: boolean;
|
|
635
|
+
prefilter_any?: string[];
|
|
636
|
+
prefilter_case_insensitive?: boolean;
|
|
637
|
+
prefilter_regex?: string;
|
|
638
|
+
prefilter_window_bytes?: number;
|
|
639
|
+
prepared_artifact_policy?: "include" | "omit";
|
|
640
|
+
};
|
|
641
|
+
type NativeSearchOptions = {
|
|
642
|
+
literal_case_insensitive?: boolean;
|
|
643
|
+
literal_whole_words?: boolean;
|
|
644
|
+
regex_whole_words?: boolean;
|
|
645
|
+
regex_overlap_all?: boolean;
|
|
646
|
+
regex_artifact_policy?: "include" | "omit";
|
|
647
|
+
fuzzy_case_insensitive?: boolean;
|
|
648
|
+
fuzzy_whole_words?: boolean;
|
|
649
|
+
fuzzy_normalize_diacritics?: boolean;
|
|
650
|
+
};
|
|
651
|
+
type NativeRegexMatchMeta = {
|
|
652
|
+
label: string;
|
|
653
|
+
score: number;
|
|
654
|
+
source_detail?: string;
|
|
655
|
+
requires_validation?: boolean;
|
|
656
|
+
validator_id?: string;
|
|
657
|
+
validator_input?: string;
|
|
658
|
+
min_byte_length?: number;
|
|
659
|
+
};
|
|
660
|
+
type NativeDenyListFilterData = {
|
|
661
|
+
stopwords: string[];
|
|
662
|
+
allow_list: string[];
|
|
663
|
+
person_stopwords: string[];
|
|
664
|
+
person_trailing_nouns: string[];
|
|
665
|
+
address_stopwords: string[];
|
|
666
|
+
address_jurisdiction_prefixes: string[];
|
|
667
|
+
street_types: string[];
|
|
668
|
+
address_component_terms: string[];
|
|
669
|
+
ambiguous_street_type_terms: string[];
|
|
670
|
+
first_names: string[];
|
|
671
|
+
generic_roles: string[];
|
|
672
|
+
number_abbrev_prefixes: string[];
|
|
673
|
+
sentence_starters: string[];
|
|
674
|
+
trailing_address_word_exclusions: string[];
|
|
675
|
+
document_heading_words: string[];
|
|
676
|
+
document_heading_ordinal_markers: string[];
|
|
677
|
+
defined_term_cues: string[];
|
|
678
|
+
signing_place_guards: NativeSigningPlaceGuardData[];
|
|
679
|
+
};
|
|
680
|
+
type NativeSigningPlaceGuardData = {
|
|
681
|
+
prefix_phrases: string[];
|
|
682
|
+
suffix_phrases: string[];
|
|
683
|
+
};
|
|
684
|
+
type NativeDenyListMatchData = {
|
|
685
|
+
labels?: string[][];
|
|
686
|
+
label_table?: string[];
|
|
687
|
+
label_indices?: number[][];
|
|
688
|
+
custom_labels?: string[][];
|
|
689
|
+
custom_label_indices?: number[][];
|
|
690
|
+
originals: string[];
|
|
691
|
+
sources?: string[][];
|
|
692
|
+
source_table?: string[];
|
|
693
|
+
source_indices?: number[][];
|
|
694
|
+
filters?: NativeDenyListFilterData;
|
|
695
|
+
};
|
|
696
|
+
type NativeTriggerStrategy = {
|
|
697
|
+
type: "to-next-comma";
|
|
698
|
+
stop_words?: string[];
|
|
699
|
+
max_length?: number;
|
|
700
|
+
} | {
|
|
701
|
+
type: "to-end-of-line";
|
|
702
|
+
} | {
|
|
703
|
+
type: "n-words";
|
|
704
|
+
count: number;
|
|
705
|
+
} | {
|
|
706
|
+
type: "company-id-value";
|
|
707
|
+
} | {
|
|
708
|
+
type: "address";
|
|
709
|
+
max_chars?: number;
|
|
710
|
+
} | {
|
|
711
|
+
type: "match-pattern";
|
|
712
|
+
pattern: string;
|
|
713
|
+
flags?: string;
|
|
714
|
+
};
|
|
715
|
+
type NativeTriggerValidation = {
|
|
716
|
+
type: "starts-uppercase";
|
|
717
|
+
} | {
|
|
718
|
+
type: "min-length";
|
|
719
|
+
min: number;
|
|
720
|
+
} | {
|
|
721
|
+
type: "max-length";
|
|
722
|
+
max: number;
|
|
723
|
+
} | {
|
|
724
|
+
type: "no-digits";
|
|
725
|
+
} | {
|
|
726
|
+
type: "has-digits";
|
|
727
|
+
} | {
|
|
728
|
+
type: "matches-pattern";
|
|
729
|
+
pattern: string;
|
|
730
|
+
flags?: string;
|
|
731
|
+
} | {
|
|
732
|
+
type: "valid-id";
|
|
733
|
+
validator: string;
|
|
734
|
+
};
|
|
735
|
+
type NativeTriggerRule = {
|
|
736
|
+
trigger: string;
|
|
737
|
+
label: string;
|
|
738
|
+
strategy: NativeTriggerStrategy;
|
|
739
|
+
validations: NativeTriggerValidation[];
|
|
740
|
+
include_trigger: boolean;
|
|
741
|
+
};
|
|
742
|
+
type NativeTriggerData = {
|
|
743
|
+
rules: NativeTriggerRule[];
|
|
744
|
+
address_stop_keywords: string[];
|
|
745
|
+
party_position_terms: string[];
|
|
746
|
+
post_nominals: string[];
|
|
747
|
+
sentence_terminal_currency_terms: string[];
|
|
748
|
+
phone_extension_labels: string[];
|
|
749
|
+
number_markers: string[];
|
|
750
|
+
number_labels: string[];
|
|
751
|
+
};
|
|
752
|
+
type NativeLegalFormData = {
|
|
753
|
+
suffixes: string[];
|
|
754
|
+
normalized_boundary_suffixes: string[];
|
|
755
|
+
normalized_in_name_words: string[];
|
|
756
|
+
normalized_suffix_words: string[];
|
|
757
|
+
role_heads: string[];
|
|
758
|
+
sentence_verb_indicators: string[];
|
|
759
|
+
clause_noun_heads: string[];
|
|
760
|
+
connector_prose_heads: string[];
|
|
761
|
+
structural_single_cap_prefixes: string[];
|
|
762
|
+
leading_clause_phrases: string[];
|
|
763
|
+
leading_clause_direct_prefixes: string[];
|
|
764
|
+
connector_words: string[];
|
|
765
|
+
and_connector_words: string[];
|
|
766
|
+
in_name_prepositions: string[];
|
|
767
|
+
company_suffix_words: string[];
|
|
768
|
+
comma_gated_direct_prefixes: string[];
|
|
769
|
+
};
|
|
770
|
+
type NativeDateData = {
|
|
771
|
+
month_names_by_language: DateMonthData;
|
|
772
|
+
year_words_by_language: YearWordData;
|
|
773
|
+
};
|
|
774
|
+
type NativeMonetaryData = MonetaryData;
|
|
775
|
+
type NativeAddressSeedData = AddressSeedData;
|
|
776
|
+
type NativeAddressContextData = AddressContextData;
|
|
777
|
+
type NativeCoreferencePatternData = {
|
|
778
|
+
pattern: string;
|
|
779
|
+
flags: string;
|
|
780
|
+
};
|
|
781
|
+
type NativeCoreferenceData = {
|
|
782
|
+
definition_patterns: NativeCoreferencePatternData[];
|
|
783
|
+
role_stop_terms: string[];
|
|
784
|
+
legal_form_aliases: string[];
|
|
785
|
+
organization_suffixes: string[];
|
|
786
|
+
organization_determiners: string[];
|
|
787
|
+
};
|
|
788
|
+
type NativeNameCorpusData = {
|
|
789
|
+
first_names: string[];
|
|
790
|
+
surnames: string[];
|
|
791
|
+
title_tokens: string[];
|
|
792
|
+
title_abbreviations: string[];
|
|
793
|
+
excluded_words: string[];
|
|
794
|
+
common_words: string[];
|
|
795
|
+
non_western_names: string[];
|
|
796
|
+
excluded_all_caps: string[];
|
|
797
|
+
ja_suffixes: string[];
|
|
798
|
+
arabic_connectors: string[];
|
|
799
|
+
relation_connectors: string[];
|
|
800
|
+
hyphenated_prefixes: string[];
|
|
801
|
+
cjk_non_person_terms: string[];
|
|
802
|
+
cjk_surname_starters: string[];
|
|
803
|
+
organization_terms: string[];
|
|
804
|
+
};
|
|
805
|
+
type NativeNameCorpusMode = "full" | "supplemental";
|
|
806
|
+
type NativeZonePatternData = {
|
|
807
|
+
pattern: string;
|
|
808
|
+
flags: string;
|
|
809
|
+
};
|
|
810
|
+
type NativeZoneSigningClauseData = {
|
|
811
|
+
prefix: string;
|
|
812
|
+
suffix: string;
|
|
813
|
+
prepositions: string[];
|
|
814
|
+
};
|
|
815
|
+
type NativeZoneData = {
|
|
816
|
+
section_heading_patterns: NativeZonePatternData[];
|
|
817
|
+
signing_clauses: NativeZoneSigningClauseData[];
|
|
818
|
+
};
|
|
819
|
+
type NativeGazetteerData = {
|
|
820
|
+
labels: string[];
|
|
821
|
+
is_fuzzy: boolean[];
|
|
822
|
+
};
|
|
823
|
+
type NativeHotwordRule = {
|
|
824
|
+
hotwords: string[];
|
|
825
|
+
target_labels: string[];
|
|
826
|
+
score_adjustment: number;
|
|
827
|
+
reclassify_to?: string;
|
|
828
|
+
proximity_before: number;
|
|
829
|
+
proximity_after: number;
|
|
830
|
+
};
|
|
831
|
+
type NativeHotwordRuleData = {
|
|
832
|
+
rules: NativeHotwordRule[];
|
|
833
|
+
pattern_rule_indices: number[];
|
|
834
|
+
};
|
|
835
|
+
type NativeSignatureData = {
|
|
836
|
+
labels: string[];
|
|
837
|
+
witness_phrases: string[];
|
|
838
|
+
name_particles: string[];
|
|
839
|
+
post_nominal_suffixes: string[];
|
|
840
|
+
organization_suffixes: string[];
|
|
841
|
+
image_stub_prefixes: string[];
|
|
842
|
+
};
|
|
843
|
+
type NativePreparedSearchConfig = {
|
|
844
|
+
regex_patterns: NativeSearchPattern[];
|
|
845
|
+
custom_regex_patterns: NativeSearchPattern[];
|
|
846
|
+
literal_patterns: NativeSearchPattern[];
|
|
847
|
+
regex_options: NativeSearchOptions;
|
|
848
|
+
custom_regex_options: NativeSearchOptions;
|
|
849
|
+
literal_options: NativeSearchOptions;
|
|
850
|
+
literal_patterns_from_deny_list_data?: boolean;
|
|
851
|
+
allowed_labels: string[];
|
|
852
|
+
threshold: number;
|
|
853
|
+
confidence_boost: boolean;
|
|
854
|
+
slices: {
|
|
855
|
+
regex: PatternSlice;
|
|
856
|
+
custom_regex: PatternSlice;
|
|
857
|
+
legal_forms?: PatternSlice;
|
|
858
|
+
triggers?: PatternSlice;
|
|
859
|
+
deny_list: PatternSlice;
|
|
860
|
+
street_types?: PatternSlice;
|
|
861
|
+
gazetteer: PatternSlice;
|
|
862
|
+
countries: PatternSlice;
|
|
863
|
+
hotwords?: PatternSlice;
|
|
864
|
+
};
|
|
865
|
+
regex_meta: NativeRegexMatchMeta[];
|
|
866
|
+
custom_regex_meta: NativeRegexMatchMeta[];
|
|
867
|
+
deny_list_data?: NativeDenyListMatchData;
|
|
868
|
+
false_positive_filters?: NativeDenyListFilterData;
|
|
869
|
+
gazetteer_data?: NativeGazetteerData;
|
|
870
|
+
country_data?: CountryData;
|
|
871
|
+
hotword_data?: NativeHotwordRuleData;
|
|
872
|
+
trigger_data?: NativeTriggerData;
|
|
873
|
+
legal_form_data?: NativeLegalFormData;
|
|
874
|
+
address_seed_data?: NativeAddressSeedData;
|
|
875
|
+
zone_data?: NativeZoneData;
|
|
876
|
+
address_context_data?: NativeAddressContextData;
|
|
877
|
+
coreference_data?: NativeCoreferenceData;
|
|
878
|
+
name_corpus_data?: NativeNameCorpusData;
|
|
879
|
+
signature_data?: NativeSignatureData;
|
|
880
|
+
name_corpus_mode?: NativeNameCorpusMode;
|
|
881
|
+
date_data?: NativeDateData;
|
|
882
|
+
monetary_data?: NativeMonetaryData;
|
|
883
|
+
};
|
|
522
884
|
type GazetteerData = {
|
|
523
885
|
/** Maps local pattern index to entry label. */labels: string[];
|
|
524
886
|
/**
|
|
@@ -547,6 +909,7 @@ type UnifiedSearchInstance = {
|
|
|
547
909
|
denyListData: DenyListData | null;
|
|
548
910
|
gazetteerData: GazetteerData | null;
|
|
549
911
|
countryData: CountryData | null;
|
|
912
|
+
nativeStaticConfig: NativePreparedSearchConfig;
|
|
550
913
|
};
|
|
551
914
|
declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: GazetteerEntry[], ctx?: PipelineContext) => Promise<UnifiedSearchInstance>;
|
|
552
915
|
//#endregion
|
|
@@ -612,6 +975,9 @@ type PipelineContext = {
|
|
|
612
975
|
search: UnifiedSearchInstance | null;
|
|
613
976
|
searchKey: string;
|
|
614
977
|
searchPromise: Promise<UnifiedSearchInstance> | null;
|
|
978
|
+
nativePipelinePackage: Uint8Array | null;
|
|
979
|
+
nativePipelinePackageKey: string;
|
|
980
|
+
nativePipelinePackagePromise: Promise<Uint8Array> | null;
|
|
615
981
|
nameCorpus: NameCorpusData | null;
|
|
616
982
|
nameCorpusKey: string;
|
|
617
983
|
nameCorpusPromise: Promise<void> | null;
|
|
@@ -621,6 +987,8 @@ type PipelineContext = {
|
|
|
621
987
|
allowListPromise: Promise<ReadonlySet<string>> | null;
|
|
622
988
|
personStopwords: ReadonlySet<string> | null;
|
|
623
989
|
personStopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
990
|
+
definedTermHeads: ReadonlySet<string> | null;
|
|
991
|
+
definedTermHeadsPromise: Promise<ReadonlySet<string>> | null;
|
|
624
992
|
addressStopwords: ReadonlySet<string> | null;
|
|
625
993
|
addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
|
|
626
994
|
firstNameExclusions: ReadonlySet<string> | null;
|
|
@@ -628,6 +996,7 @@ type PipelineContext = {
|
|
|
628
996
|
genericRoles: ReadonlySet<string> | null;
|
|
629
997
|
genericRolesPromise: Promise<ReadonlySet<string>> | null;
|
|
630
998
|
corefPatterns: DefinitionPattern[] | null;
|
|
999
|
+
corefPatternsKey: string;
|
|
631
1000
|
corefPatternsPromise: Promise<DefinitionPattern[]> | null;
|
|
632
1001
|
corefLoadAttempted: boolean;
|
|
633
1002
|
roleStopSet: ReadonlySet<string> | null;
|
|
@@ -695,21 +1064,304 @@ type PipelineOptions = {
|
|
|
695
1064
|
*/
|
|
696
1065
|
declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
|
|
697
1066
|
//#endregion
|
|
1067
|
+
//#region src/native.d.ts
|
|
1068
|
+
type NativeBindingOperatorConfig = {
|
|
1069
|
+
operators?: Record<string, OperatorType>;
|
|
1070
|
+
redactString?: string;
|
|
1071
|
+
};
|
|
1072
|
+
type NativeDiagnosticsBatchCallback = (diagnosticsJson: string) => void;
|
|
1073
|
+
type NativeResultEventCallback = (eventJson: string) => void;
|
|
1074
|
+
type NativeBindingRedactionEntry = {
|
|
1075
|
+
placeholder: string;
|
|
1076
|
+
original: string;
|
|
1077
|
+
};
|
|
1078
|
+
type NativeBindingOperatorEntry = {
|
|
1079
|
+
placeholder: string;
|
|
1080
|
+
operator: OperatorType;
|
|
1081
|
+
};
|
|
1082
|
+
type NativeBindingPipelineEntity = {
|
|
1083
|
+
start: number;
|
|
1084
|
+
end: number;
|
|
1085
|
+
label: string;
|
|
1086
|
+
text: string;
|
|
1087
|
+
score: number;
|
|
1088
|
+
source: string;
|
|
1089
|
+
sourceDetail?: string | null;
|
|
1090
|
+
};
|
|
1091
|
+
type NativeBindingRedactionResult = {
|
|
1092
|
+
redactedText: string;
|
|
1093
|
+
redactionMap: NativeBindingRedactionEntry[];
|
|
1094
|
+
operatorMap: NativeBindingOperatorEntry[];
|
|
1095
|
+
entityCount: number;
|
|
1096
|
+
};
|
|
1097
|
+
type NativeBindingStaticRedactionResult = {
|
|
1098
|
+
resolvedEntities: NativeBindingPipelineEntity[];
|
|
1099
|
+
redaction: NativeBindingRedactionResult;
|
|
1100
|
+
};
|
|
1101
|
+
type NativePreparedSearchBinding = {
|
|
1102
|
+
prepareDiagnosticsJson?: () => string;
|
|
1103
|
+
warmLazyRegex?: () => void;
|
|
1104
|
+
warm_lazy_regex?: () => void;
|
|
1105
|
+
warmLazyRegexDiagnosticsJson?: () => string;
|
|
1106
|
+
warm_lazy_regex_diagnostics_json?: () => string;
|
|
1107
|
+
redactStaticEntities: (fullText: string, operators?: NativeBindingOperatorConfig) => NativeBindingStaticRedactionResult;
|
|
1108
|
+
redactStaticEntitiesJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
1109
|
+
redactStaticEntitiesResultStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onEvent: NativeResultEventCallback) => string;
|
|
1110
|
+
redactStaticEntitiesDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
1111
|
+
redactStaticEntitiesDiagnosticsStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onBatch: NativeDiagnosticsBatchCallback) => string;
|
|
1112
|
+
redactStaticEntitiesSummaryDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
1113
|
+
};
|
|
1114
|
+
type NativeAnonymizeBinding = {
|
|
1115
|
+
normalizeForSearch: (text: string) => string;
|
|
1116
|
+
nativePackageVersion: () => string;
|
|
1117
|
+
NativePreparedSearch: {
|
|
1118
|
+
fromConfigJsonBytes: (configJson: Uint8Array) => NativePreparedSearchBinding;
|
|
1119
|
+
fromPreparedPackageBytes: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1120
|
+
fromPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1121
|
+
fromTrustedPreparedPackageBytes?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1122
|
+
fromTrustedPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1123
|
+
};
|
|
1124
|
+
prepareStaticSearchPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
1125
|
+
prepareStaticSearchCompressedPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
1126
|
+
};
|
|
1127
|
+
type NativeOperatorConfig = {
|
|
1128
|
+
operators?: Record<string, OperatorType>;
|
|
1129
|
+
redactString?: string;
|
|
1130
|
+
};
|
|
1131
|
+
type NativePipelineEntity = {
|
|
1132
|
+
start: number;
|
|
1133
|
+
end: number;
|
|
1134
|
+
label: string;
|
|
1135
|
+
text: string;
|
|
1136
|
+
score: number;
|
|
1137
|
+
source: string;
|
|
1138
|
+
sourceDetail?: string;
|
|
1139
|
+
};
|
|
1140
|
+
type NativeRedactionResult = {
|
|
1141
|
+
redactedText: string;
|
|
1142
|
+
redactionMap: Map<string, string>;
|
|
1143
|
+
operatorMap: Map<string, OperatorType>;
|
|
1144
|
+
entityCount: number;
|
|
1145
|
+
};
|
|
1146
|
+
type NativeStaticRedactionResult = {
|
|
1147
|
+
resolvedEntities: NativePipelineEntity[];
|
|
1148
|
+
redaction: NativeRedactionResult;
|
|
1149
|
+
};
|
|
1150
|
+
type NativeSearchPackageOptions = {
|
|
1151
|
+
binding: NativeAnonymizeBinding;
|
|
1152
|
+
config: NativePreparedSearchConfig;
|
|
1153
|
+
compressed?: boolean;
|
|
1154
|
+
};
|
|
1155
|
+
type NativeSearchPackageInput = NativePreparedSearchConfig | string | Uint8Array;
|
|
1156
|
+
type SharedNativeSearchPackageOptions = {
|
|
1157
|
+
binding: NativeAnonymizeBinding;
|
|
1158
|
+
config: NativeSearchPackageInput;
|
|
1159
|
+
compressed?: boolean;
|
|
1160
|
+
};
|
|
1161
|
+
type SharedNativePreparedPackageOptions = {
|
|
1162
|
+
binding: NativeAnonymizeBinding;
|
|
1163
|
+
packageBytes: Uint8Array;
|
|
1164
|
+
};
|
|
1165
|
+
type SharedNativeRedactTextJsonOptions = {
|
|
1166
|
+
binding: NativeAnonymizeBinding;
|
|
1167
|
+
config: NativeSearchPackageInput;
|
|
1168
|
+
fullText: string;
|
|
1169
|
+
operators?: NativeOperatorConfig;
|
|
1170
|
+
};
|
|
1171
|
+
type SharedNativeRedactTextOptions = SharedNativeRedactTextJsonOptions;
|
|
1172
|
+
type SharedNativeDiagnosticsJsonOptions = SharedNativeRedactTextJsonOptions;
|
|
1173
|
+
type SharedNativeRedactTextStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
1174
|
+
onEvent: NativeResultEventCallback;
|
|
1175
|
+
};
|
|
1176
|
+
type NativeNormalizeOptions = {
|
|
1177
|
+
binding: NativeAnonymizeBinding;
|
|
1178
|
+
text: string;
|
|
1179
|
+
};
|
|
1180
|
+
type NativeAnonymizerFromConfigOptions = {
|
|
1181
|
+
binding: NativeAnonymizeBinding;
|
|
1182
|
+
config: NativePreparedSearchConfig;
|
|
1183
|
+
};
|
|
1184
|
+
type NativeAnonymizerFromPackageOptions = {
|
|
1185
|
+
binding: NativeAnonymizeBinding;
|
|
1186
|
+
packageBytes: Uint8Array;
|
|
1187
|
+
};
|
|
1188
|
+
type NativePipelineFromPackageOptions = NativeAnonymizerFromPackageOptions;
|
|
1189
|
+
type NativeBindingVersionOptions = {
|
|
1190
|
+
binding: NativeAnonymizeBinding;
|
|
1191
|
+
expectedVersion: string;
|
|
1192
|
+
};
|
|
1193
|
+
declare class PreparedNativeAnonymizer {
|
|
1194
|
+
#private;
|
|
1195
|
+
constructor(prepared: NativePreparedSearchBinding);
|
|
1196
|
+
prepareDiagnosticsJson(): string | null;
|
|
1197
|
+
prepare_diagnostics_json(): string | null;
|
|
1198
|
+
warmLazyRegex(): void;
|
|
1199
|
+
warm_lazy_regex(): void;
|
|
1200
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
1201
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
1202
|
+
redactStaticEntities(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1203
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1204
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1205
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1206
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1207
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1208
|
+
redactStaticEntitiesDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1209
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1210
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1211
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1212
|
+
redactStaticEntitiesSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1213
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1214
|
+
}
|
|
1215
|
+
declare class PreparedNativePipeline {
|
|
1216
|
+
#private;
|
|
1217
|
+
constructor(anonymizer: PreparedNativeAnonymizer);
|
|
1218
|
+
prepareDiagnosticsJson(): string | null;
|
|
1219
|
+
prepare_diagnostics_json(): string | null;
|
|
1220
|
+
warmLazyRegex(): void;
|
|
1221
|
+
warm_lazy_regex(): void;
|
|
1222
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
1223
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
1224
|
+
redactText(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1225
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1226
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1227
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1228
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1229
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1230
|
+
redactTextDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1231
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1232
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1233
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1234
|
+
redactTextSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1235
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1236
|
+
}
|
|
1237
|
+
declare const encodeNativeSearchConfig: (config: NativePreparedSearchConfig) => Uint8Array;
|
|
1238
|
+
declare const encodeNativeSearchConfigInput: (config: NativeSearchPackageInput) => Uint8Array;
|
|
1239
|
+
declare const getNativeBindingVersion: (binding: NativeAnonymizeBinding) => string;
|
|
1240
|
+
declare const native_package_version: (binding: NativeAnonymizeBinding) => string;
|
|
1241
|
+
declare const normalize_for_search: ({
|
|
1242
|
+
binding,
|
|
1243
|
+
text
|
|
1244
|
+
}: NativeNormalizeOptions) => string;
|
|
1245
|
+
declare const assertNativeBindingVersion: ({
|
|
1246
|
+
binding,
|
|
1247
|
+
expectedVersion
|
|
1248
|
+
}: NativeBindingVersionOptions) => void;
|
|
1249
|
+
declare const prepareNativeSearchPackage: ({
|
|
1250
|
+
binding,
|
|
1251
|
+
config,
|
|
1252
|
+
compressed
|
|
1253
|
+
}: NativeSearchPackageOptions) => Uint8Array;
|
|
1254
|
+
declare const prepare_search_package: ({
|
|
1255
|
+
binding,
|
|
1256
|
+
config,
|
|
1257
|
+
compressed
|
|
1258
|
+
}: SharedNativeSearchPackageOptions) => Uint8Array;
|
|
1259
|
+
declare const createNativeAnonymizerFromConfig: ({
|
|
1260
|
+
binding,
|
|
1261
|
+
config
|
|
1262
|
+
}: NativeAnonymizerFromConfigOptions) => PreparedNativeAnonymizer;
|
|
1263
|
+
declare const createNativeAnonymizerFromPackage: ({
|
|
1264
|
+
binding,
|
|
1265
|
+
packageBytes
|
|
1266
|
+
}: NativeAnonymizerFromPackageOptions) => PreparedNativeAnonymizer;
|
|
1267
|
+
declare const load_prepared_package: ({
|
|
1268
|
+
binding,
|
|
1269
|
+
packageBytes
|
|
1270
|
+
}: SharedNativePreparedPackageOptions) => PreparedNativeAnonymizer;
|
|
1271
|
+
declare const redact_text_json: ({
|
|
1272
|
+
binding,
|
|
1273
|
+
config,
|
|
1274
|
+
fullText,
|
|
1275
|
+
operators
|
|
1276
|
+
}: SharedNativeRedactTextJsonOptions) => string;
|
|
1277
|
+
declare const redact_text: ({
|
|
1278
|
+
binding,
|
|
1279
|
+
config,
|
|
1280
|
+
fullText,
|
|
1281
|
+
operators
|
|
1282
|
+
}: SharedNativeRedactTextOptions) => NativeStaticRedactionResult;
|
|
1283
|
+
declare const redact_text_stream_json: ({
|
|
1284
|
+
binding,
|
|
1285
|
+
config,
|
|
1286
|
+
fullText,
|
|
1287
|
+
operators,
|
|
1288
|
+
onEvent
|
|
1289
|
+
}: SharedNativeRedactTextStreamJsonOptions) => string | null;
|
|
1290
|
+
declare const diagnostics_json: ({
|
|
1291
|
+
binding,
|
|
1292
|
+
config,
|
|
1293
|
+
fullText,
|
|
1294
|
+
operators
|
|
1295
|
+
}: SharedNativeDiagnosticsJsonOptions) => string | null;
|
|
1296
|
+
declare const createNativePipelineFromPackage: ({
|
|
1297
|
+
binding,
|
|
1298
|
+
packageBytes
|
|
1299
|
+
}: NativePipelineFromPackageOptions) => PreparedNativePipeline;
|
|
1300
|
+
declare const PreparedSearch: typeof PreparedNativeAnonymizer;
|
|
1301
|
+
type PreparedSearch = PreparedNativeAnonymizer;
|
|
1302
|
+
//#endregion
|
|
1303
|
+
//#region src/native-default-config.d.ts
|
|
1304
|
+
declare const DEFAULT_NATIVE_PIPELINE_CONFIG: PipelineConfig;
|
|
1305
|
+
//#endregion
|
|
1306
|
+
//#region src/native-pipeline.d.ts
|
|
1307
|
+
type NativePipelineUnsupportedFeature = "enableNer";
|
|
1308
|
+
type NativePipelineCompatibility = {
|
|
1309
|
+
status: "supported";
|
|
1310
|
+
} | {
|
|
1311
|
+
status: "unsupported";
|
|
1312
|
+
unsupportedFeatures: NativePipelineUnsupportedFeature[];
|
|
1313
|
+
};
|
|
1314
|
+
type NativePipelineBuildOptions = {
|
|
1315
|
+
binding: NativeAnonymizeBinding;
|
|
1316
|
+
config: PipelineConfig;
|
|
1317
|
+
gazetteerEntries?: GazetteerEntry[];
|
|
1318
|
+
context?: PipelineContext;
|
|
1319
|
+
};
|
|
1320
|
+
type NativePipelinePackageOptions = NativePipelineBuildOptions & {
|
|
1321
|
+
compressed?: boolean;
|
|
1322
|
+
};
|
|
1323
|
+
declare const getNativePipelineCompatibility: (config: PipelineConfig) => NativePipelineCompatibility;
|
|
1324
|
+
declare const assertNativePipelineSupported: (config: PipelineConfig) => void;
|
|
1325
|
+
declare const prepareNativePipelineConfig: ({
|
|
1326
|
+
config,
|
|
1327
|
+
gazetteerEntries,
|
|
1328
|
+
context
|
|
1329
|
+
}: Omit<NativePipelineBuildOptions, "binding">) => Promise<NativePreparedSearchConfig>;
|
|
1330
|
+
declare const prepareNativePipelinePackage: ({
|
|
1331
|
+
binding,
|
|
1332
|
+
config,
|
|
1333
|
+
gazetteerEntries,
|
|
1334
|
+
context,
|
|
1335
|
+
compressed
|
|
1336
|
+
}: NativePipelinePackageOptions) => Promise<Uint8Array>;
|
|
1337
|
+
declare const createNativePipelineFromConfig: ({
|
|
1338
|
+
binding,
|
|
1339
|
+
config,
|
|
1340
|
+
gazetteerEntries,
|
|
1341
|
+
context
|
|
1342
|
+
}: NativePipelineBuildOptions) => Promise<PreparedNativePipeline>;
|
|
1343
|
+
//#endregion
|
|
698
1344
|
//#region src/redact.d.ts
|
|
699
1345
|
/**
|
|
700
1346
|
* Build a stable mapping from entity text to numbered
|
|
701
1347
|
* placeholders. Same real-world value always maps to the
|
|
702
1348
|
* same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
|
|
703
|
-
*
|
|
1349
|
+
* share one person placeholder).
|
|
704
1350
|
*
|
|
705
|
-
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
706
|
-
*
|
|
1351
|
+
* Placeholder format: [LABEL_N] where LABEL is uppercase.
|
|
1352
|
+
* N is allocated per label and skips tokens already present
|
|
1353
|
+
* in reserved text.
|
|
707
1354
|
*
|
|
708
1355
|
* @param _ctx Unused. Kept for signature compatibility;
|
|
709
1356
|
* coref alias links now travel on the entities
|
|
710
1357
|
* themselves (`corefSourceText`).
|
|
711
1358
|
*/
|
|
712
|
-
|
|
1359
|
+
type PlaceholderMapOptions = {
|
|
1360
|
+
reservedText?: string;
|
|
1361
|
+
};
|
|
1362
|
+
declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext, {
|
|
1363
|
+
reservedText
|
|
1364
|
+
}?: PlaceholderMapOptions) => Map<string, string>;
|
|
713
1365
|
/**
|
|
714
1366
|
* Apply redactions to the source text, replacing each
|
|
715
1367
|
* confirmed entity span using the configured operator.
|
|
@@ -781,7 +1433,7 @@ type TriggerPatterns = {
|
|
|
781
1433
|
patterns: string[];
|
|
782
1434
|
rules: TriggerRule[];
|
|
783
1435
|
};
|
|
784
|
-
declare const buildTriggerPatterns: () => Promise<TriggerPatterns>;
|
|
1436
|
+
declare const buildTriggerPatterns: (languages?: readonly string[]) => Promise<TriggerPatterns>;
|
|
785
1437
|
/**
|
|
786
1438
|
* Process trigger matches from the unified search.
|
|
787
1439
|
* Receives all matches; filters to the trigger slice
|
|
@@ -791,20 +1443,6 @@ declare const buildTriggerPatterns: () => Promise<TriggerPatterns>;
|
|
|
791
1443
|
*/
|
|
792
1444
|
declare const processTriggerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, rules: readonly TriggerRule[]) => Entity[];
|
|
793
1445
|
//#endregion
|
|
794
|
-
//#region src/detectors/address-seeds.d.ts
|
|
795
|
-
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
796
|
-
/**
|
|
797
|
-
* Process address seeds from the unified search.
|
|
798
|
-
* Receives all matches; filters to the street types
|
|
799
|
-
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
800
|
-
* existingEntities for seed collection, clustering,
|
|
801
|
-
* expansion, and scoring.
|
|
802
|
-
*
|
|
803
|
-
* Runs as a post-processor after all other detectors,
|
|
804
|
-
* using their output as seed sources.
|
|
805
|
-
*/
|
|
806
|
-
declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
|
|
807
|
-
//#endregion
|
|
808
1446
|
//#region src/detectors/gazetteer.d.ts
|
|
809
1447
|
/**
|
|
810
1448
|
* Build TextSearch-compatible patterns from gazetteer
|
|
@@ -836,13 +1474,16 @@ declare const buildGazetteerPatterns: (entries: GazetteerEntry[]) => {
|
|
|
836
1474
|
declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: GazetteerData) => Entity[];
|
|
837
1475
|
//#endregion
|
|
838
1476
|
//#region src/detectors/coreference.d.ts
|
|
1477
|
+
type ExtractDefinedTermsOptions = {
|
|
1478
|
+
languages?: readonly string[];
|
|
1479
|
+
};
|
|
839
1480
|
type DefinedTerm = {
|
|
840
1481
|
alias: string;
|
|
841
1482
|
label: string; /** Position of the definition in the source text */
|
|
842
1483
|
definitionStart: number; /** Original entity text the alias refers to */
|
|
843
1484
|
sourceText: string;
|
|
844
1485
|
};
|
|
845
|
-
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext) => Promise<DefinedTerm[]>;
|
|
1486
|
+
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext, options?: ExtractDefinedTermsOptions) => Promise<DefinedTerm[]>;
|
|
846
1487
|
/**
|
|
847
1488
|
* Find all occurrences of defined-term aliases in the
|
|
848
1489
|
* full text. Returns Entity spans for each match.
|
|
@@ -979,21 +1620,6 @@ declare const initAddressComponents: () => Promise<void>;
|
|
|
979
1620
|
*/
|
|
980
1621
|
declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext, fullText?: string) => Entity[];
|
|
981
1622
|
//#endregion
|
|
982
|
-
//#region src/filters/confidence-boost.d.ts
|
|
983
|
-
/**
|
|
984
|
-
* Boost confidence of near-miss NER entities that appear
|
|
985
|
-
* near high-confidence detections (regex, trigger phrase).
|
|
986
|
-
*
|
|
987
|
-
* If an NER entity scored between (threshold - 0.15) and
|
|
988
|
-
* threshold, count how many confirmed entities exist within
|
|
989
|
-
* a 150-char window. Add +0.05 per co-located entity.
|
|
990
|
-
* If the boosted score crosses the threshold, include it.
|
|
991
|
-
*
|
|
992
|
-
* Only mutates score on near-miss entities; high-confidence
|
|
993
|
-
* entities pass through unchanged.
|
|
994
|
-
*/
|
|
995
|
-
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
996
|
-
//#endregion
|
|
997
1623
|
//#region src/filters/hotword-rules.d.ts
|
|
998
1624
|
type HotwordRule = {
|
|
999
1625
|
hotwords: string[];
|
|
@@ -1213,5 +1839,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
|
|
|
1213
1839
|
*/
|
|
1214
1840
|
declare const normalizeForSearch: (text: string) => string;
|
|
1215
1841
|
//#endregion
|
|
1216
|
-
export { type AnonymisationOperator, CURRENCY_PATTERN_META, type ChunkSpan, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, chunkTextWithOffsets, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
1842
|
+
export { type AnonymisationOperator, CURRENCY_PATTERN_META, type ChunkSpan, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NativeAnonymizeBinding, type NativeAnonymizerFromConfigOptions, type NativeAnonymizerFromPackageOptions, type NativeBindingVersionOptions, type NativeNormalizeOptions, type NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, type NativePipelineEntity, type NativePipelineFromPackageOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, type NativePreparedSearchBinding, type NativeRedactionResult, type NativeResultEventCallback, type NativeSearchPackageInput, type NativeSearchPackageOptions, type NativeStaticRedactionResult, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, type PreparedSearch as PreparedSearchInstance, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type SharedNativeDiagnosticsJsonOptions, type SharedNativePreparedPackageOptions, type SharedNativeRedactTextJsonOptions, type SharedNativeRedactTextOptions, type SharedNativeRedactTextStreamJsonOptions, type SharedNativeSearchPackageOptions, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, assertNativeBindingVersion, assertNativePipelineSupported, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, chunkTextWithOffsets, classifyZones, computeChunkOffsets, corefKey, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromPackage, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, diagnostics_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, getNativeBindingVersion, getNativePipelineCompatibility, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, load_prepared_package, mergeAndDedup, mergeChunkEntities, native_package_version, normalizeForSearch, normalize_for_search, prepareBatch, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, preparePipelineSearch, prepare_search_package, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, redact_text, redact_text_json, redact_text_stream_json, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
1217
1843
|
//# sourceMappingURL=wasm.d.mts.map
|