@stll/anonymize-wasm 1.4.11 → 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -2
- package/dist/address-boundaries.mjs +2 -0
- package/dist/address-jurisdiction-prefixes.mjs +16 -0
- package/dist/address-jurisdiction-prefixes.mjs.map +1 -0
- package/dist/address-stop-keywords.mjs +12 -1
- package/dist/address-stopwords.mjs +2 -0
- package/dist/address-unit-abbreviations.mjs +15 -0
- package/dist/address-unit-abbreviations.mjs.map +1 -0
- package/dist/allow-list.mjs +1 -0
- package/dist/clause-noun-heads.mjs +5 -1
- package/dist/constants.mjs +1 -0
- package/dist/constants.mjs.map +1 -1
- package/dist/constants2.d.mts +1 -1
- package/dist/coreference-org-determiners.mjs +19 -0
- package/dist/coreference-org-determiners.mjs.map +1 -0
- package/dist/defined-term-heads.mjs +15 -0
- package/dist/defined-term-heads.mjs.map +1 -0
- package/dist/false-positive-shapes.mjs +36 -0
- package/dist/false-positive-shapes.mjs.map +1 -0
- package/dist/legal-role-heads.cs.mjs +6 -0
- package/dist/organization-unit-heads.mjs +20 -0
- package/dist/organization-unit-heads.mjs.map +1 -0
- package/dist/person-stopwords.mjs +205 -199
- package/dist/signing-clauses.mjs +33 -9
- package/dist/wasm.d.mts +705 -45
- package/dist/wasm.mjs +4225 -1442
- package/dist/wasm.mjs.map +1 -1
- package/package.json +3 -3
- package/dist/address-prepositions.mjs +0 -182
- package/dist/address-prepositions.mjs.map +0 -1
package/dist/wasm.d.mts
CHANGED
|
@@ -180,6 +180,7 @@ type CompiledValidation = {
|
|
|
180
180
|
re: RegExp;
|
|
181
181
|
} | {
|
|
182
182
|
type: "valid-id";
|
|
183
|
+
validator: ValidIdValidator;
|
|
183
184
|
check: (value: string) => boolean;
|
|
184
185
|
};
|
|
185
186
|
/**
|
|
@@ -257,6 +258,7 @@ type CustomRegexPattern = {
|
|
|
257
258
|
pattern: string;
|
|
258
259
|
label: string;
|
|
259
260
|
score?: number;
|
|
261
|
+
preparedArtifactPolicy?: "include" | "omit";
|
|
260
262
|
};
|
|
261
263
|
/**
|
|
262
264
|
* Pre-loaded dictionary data for dependency injection.
|
|
@@ -321,6 +323,18 @@ type PipelineConfig = {
|
|
|
321
323
|
threshold: number;
|
|
322
324
|
enableTriggerPhrases: boolean;
|
|
323
325
|
enableRegex: boolean;
|
|
326
|
+
/**
|
|
327
|
+
* Expected content language codes. When present, these
|
|
328
|
+
* derive default dictionary scopes for name corpus and
|
|
329
|
+
* deny-list matching unless the lower-level scope fields
|
|
330
|
+
* below are set explicitly.
|
|
331
|
+
*/
|
|
332
|
+
languages?: string[];
|
|
333
|
+
/**
|
|
334
|
+
* Convenience form for single-language documents. Ignored
|
|
335
|
+
* when `languages` is also provided.
|
|
336
|
+
*/
|
|
337
|
+
language?: string;
|
|
324
338
|
/**
|
|
325
339
|
* Enables legal-form organization detection.
|
|
326
340
|
* Required for typed callers; legacy untyped
|
|
@@ -391,21 +405,48 @@ type PipelineConfig = {
|
|
|
391
405
|
type RegexMeta = {
|
|
392
406
|
label: string;
|
|
393
407
|
score: number;
|
|
394
|
-
sourceDetail?: Entity["sourceDetail"];
|
|
408
|
+
sourceDetail?: Entity["sourceDetail"];
|
|
409
|
+
minByteLength?: number; /** Post-match stdnum validator for confirmation. */
|
|
395
410
|
validator?: Validator;
|
|
411
|
+
validatorId?: string; /** Extract the identifier portion when context is part of the regex span. */
|
|
412
|
+
validatorInput?: (text: string) => string;
|
|
413
|
+
validatorInputKind?: "digits-only" | "crypto-wallet-candidate";
|
|
396
414
|
};
|
|
397
415
|
/** Flat pattern array for text-search. */
|
|
398
416
|
declare const REGEX_PATTERNS: readonly string[];
|
|
399
417
|
/** Parallel metadata. Index = pattern index. */
|
|
400
418
|
declare const REGEX_META: readonly RegexMeta[];
|
|
419
|
+
type DateMonthData = Record<string, string[]>;
|
|
420
|
+
type YearWordData = Record<string, string[]>;
|
|
401
421
|
/**
|
|
402
422
|
* Get dynamically built date patterns from
|
|
403
423
|
* date-months.json. Returns a cached promise; the JSON
|
|
404
424
|
* is loaded only once.
|
|
405
425
|
*/
|
|
406
|
-
declare const getDatePatterns: () => Promise<string[]>;
|
|
426
|
+
declare const getDatePatterns: (languages?: readonly string[]) => Promise<string[]>;
|
|
407
427
|
/** Date pattern metadata (all are score 1 dates). */
|
|
408
428
|
declare const DATE_PATTERN_META: Readonly<RegexMeta>;
|
|
429
|
+
type MonetaryData = {
|
|
430
|
+
currencies: {
|
|
431
|
+
codes: string[];
|
|
432
|
+
symbols: string[];
|
|
433
|
+
local_names: string[];
|
|
434
|
+
};
|
|
435
|
+
amount_words: {
|
|
436
|
+
written_amount_patterns: Array<{
|
|
437
|
+
keywords: string[];
|
|
438
|
+
}>;
|
|
439
|
+
magnitude_suffixes: Array<{
|
|
440
|
+
words: string[];
|
|
441
|
+
abbreviations_case_insensitive: string[];
|
|
442
|
+
abbreviations_case_sensitive: string[];
|
|
443
|
+
}>;
|
|
444
|
+
share_quantity_terms: Array<{
|
|
445
|
+
modifiers: string[];
|
|
446
|
+
nouns: string[];
|
|
447
|
+
}>;
|
|
448
|
+
};
|
|
449
|
+
};
|
|
409
450
|
/**
|
|
410
451
|
* Get dynamically built monetary amount patterns from
|
|
411
452
|
* currencies.json. Returns a cached promise; the JSON
|
|
@@ -429,6 +470,30 @@ declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sli
|
|
|
429
470
|
//#endregion
|
|
430
471
|
//#region src/detectors/deny-list.d.ts
|
|
431
472
|
type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "nameCorpusLanguages" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "customDenyList" | "dictionaries" | "enableCountries">;
|
|
473
|
+
type DenyListFilterData = {
|
|
474
|
+
stopwords: string[];
|
|
475
|
+
allowList: string[];
|
|
476
|
+
personStopwords: string[];
|
|
477
|
+
personTrailingNouns: string[];
|
|
478
|
+
addressStopwords: string[];
|
|
479
|
+
addressJurisdictionPrefixes: string[];
|
|
480
|
+
streetTypes: string[];
|
|
481
|
+
addressComponentTerms: string[];
|
|
482
|
+
ambiguousStreetTypeTerms: string[];
|
|
483
|
+
firstNames: string[];
|
|
484
|
+
genericRoles: string[];
|
|
485
|
+
numberAbbrevPrefixes: string[];
|
|
486
|
+
sentenceStarters: string[];
|
|
487
|
+
trailingAddressWordExclusions: string[];
|
|
488
|
+
documentHeadingWords: string[];
|
|
489
|
+
documentHeadingOrdinalMarkers: string[];
|
|
490
|
+
definedTermCues: string[];
|
|
491
|
+
signingPlaceGuards: DenyListSigningPlaceGuardData[];
|
|
492
|
+
};
|
|
493
|
+
type DenyListSigningPlaceGuardData = {
|
|
494
|
+
prefixPhrases: string[];
|
|
495
|
+
suffixPhrases: string[];
|
|
496
|
+
};
|
|
432
497
|
/**
|
|
433
498
|
* Source tag for each pattern in the automaton.
|
|
434
499
|
* "deny-list" = standard deny list entry
|
|
@@ -458,6 +523,7 @@ type DenyListData = {
|
|
|
458
523
|
customLabels: (PatternLabels | undefined)[]; /** Maps pattern index → original pattern text. */
|
|
459
524
|
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
460
525
|
sources: PatternSources[];
|
|
526
|
+
filters: DenyListFilterData;
|
|
461
527
|
};
|
|
462
528
|
/**
|
|
463
529
|
* Resolve which dictionaries to load based on country
|
|
@@ -495,6 +561,25 @@ declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Diction
|
|
|
495
561
|
*/
|
|
496
562
|
declare const processDenyListMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: DenyListData, ctx?: PipelineContext) => Entity[];
|
|
497
563
|
//#endregion
|
|
564
|
+
//#region src/detectors/address-seeds.d.ts
|
|
565
|
+
type AddressSeedData = {
|
|
566
|
+
boundary_words: string[];
|
|
567
|
+
br_cep_cue_words: string[];
|
|
568
|
+
unit_abbreviations: string[];
|
|
569
|
+
};
|
|
570
|
+
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
571
|
+
/**
|
|
572
|
+
* Process address seeds from the unified search.
|
|
573
|
+
* Receives all matches; filters to the street types
|
|
574
|
+
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
575
|
+
* existingEntities for seed collection, clustering,
|
|
576
|
+
* expansion, and scoring.
|
|
577
|
+
*
|
|
578
|
+
* Runs as a post-processor after all other detectors,
|
|
579
|
+
* using their output as seed sources.
|
|
580
|
+
*/
|
|
581
|
+
declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
|
|
582
|
+
//#endregion
|
|
498
583
|
//#region src/detectors/countries.d.ts
|
|
499
584
|
/**
|
|
500
585
|
* Pre-built country patterns + parallel label/source
|
|
@@ -513,11 +598,289 @@ type CountryData = {
|
|
|
513
598
|
};
|
|
514
599
|
type CountryVariant = "name" | "alias" | "alpha3" | "alpha2";
|
|
515
600
|
//#endregion
|
|
601
|
+
//#region src/filters/confidence-boost.d.ts
|
|
602
|
+
/**
|
|
603
|
+
* Boost confidence of near-miss NER entities that appear
|
|
604
|
+
* near high-confidence detections (regex, trigger phrase).
|
|
605
|
+
*
|
|
606
|
+
* If an NER entity scored between (threshold - 0.15) and
|
|
607
|
+
* threshold, count how many confirmed entities exist within
|
|
608
|
+
* a 150-char window. Add +0.05 per co-located entity.
|
|
609
|
+
* If the boosted score crosses the threshold, include it.
|
|
610
|
+
*
|
|
611
|
+
* Only mutates score on near-miss entities; high-confidence
|
|
612
|
+
* entities pass through unchanged.
|
|
613
|
+
*/
|
|
614
|
+
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
615
|
+
type AddressContextData = {
|
|
616
|
+
address_prepositions: string[];
|
|
617
|
+
temporal_prepositions: string[];
|
|
618
|
+
street_abbreviations: string[];
|
|
619
|
+
bare_house_stopwords: string[];
|
|
620
|
+
};
|
|
621
|
+
//#endregion
|
|
516
622
|
//#region src/build-unified-search.d.ts
|
|
517
623
|
type PatternSlice = {
|
|
518
624
|
start: number;
|
|
519
625
|
end: number;
|
|
520
626
|
};
|
|
627
|
+
type NativeSearchPatternKind = "literal" | "literal-with-options" | "regex" | "fuzzy";
|
|
628
|
+
type NativeSearchPattern = {
|
|
629
|
+
kind: NativeSearchPatternKind;
|
|
630
|
+
pattern: string;
|
|
631
|
+
distance?: number;
|
|
632
|
+
case_insensitive?: boolean;
|
|
633
|
+
whole_words?: boolean;
|
|
634
|
+
lazy?: boolean;
|
|
635
|
+
prefilter_any?: string[];
|
|
636
|
+
prefilter_case_insensitive?: boolean;
|
|
637
|
+
prefilter_regex?: string;
|
|
638
|
+
prefilter_window_bytes?: number;
|
|
639
|
+
prepared_artifact_policy?: "include" | "omit";
|
|
640
|
+
};
|
|
641
|
+
type NativeSearchOptions = {
|
|
642
|
+
literal_case_insensitive?: boolean;
|
|
643
|
+
literal_whole_words?: boolean;
|
|
644
|
+
regex_whole_words?: boolean;
|
|
645
|
+
regex_overlap_all?: boolean;
|
|
646
|
+
regex_artifact_policy?: "include" | "omit";
|
|
647
|
+
fuzzy_case_insensitive?: boolean;
|
|
648
|
+
fuzzy_whole_words?: boolean;
|
|
649
|
+
fuzzy_normalize_diacritics?: boolean;
|
|
650
|
+
};
|
|
651
|
+
type NativeRegexMatchMeta = {
|
|
652
|
+
label: string;
|
|
653
|
+
score: number;
|
|
654
|
+
source_detail?: string;
|
|
655
|
+
requires_validation?: boolean;
|
|
656
|
+
validator_id?: string;
|
|
657
|
+
validator_input?: string;
|
|
658
|
+
min_byte_length?: number;
|
|
659
|
+
};
|
|
660
|
+
type NativeDenyListFilterData = {
|
|
661
|
+
stopwords: string[];
|
|
662
|
+
allow_list: string[];
|
|
663
|
+
person_stopwords: string[];
|
|
664
|
+
person_trailing_nouns: string[];
|
|
665
|
+
address_stopwords: string[];
|
|
666
|
+
address_jurisdiction_prefixes: string[];
|
|
667
|
+
street_types: string[];
|
|
668
|
+
address_component_terms: string[];
|
|
669
|
+
ambiguous_street_type_terms: string[];
|
|
670
|
+
first_names: string[];
|
|
671
|
+
generic_roles: string[];
|
|
672
|
+
number_abbrev_prefixes: string[];
|
|
673
|
+
sentence_starters: string[];
|
|
674
|
+
trailing_address_word_exclusions: string[];
|
|
675
|
+
document_heading_words: string[];
|
|
676
|
+
document_heading_ordinal_markers: string[];
|
|
677
|
+
defined_term_cues: string[];
|
|
678
|
+
signing_place_guards: NativeSigningPlaceGuardData[];
|
|
679
|
+
};
|
|
680
|
+
type NativeSigningPlaceGuardData = {
|
|
681
|
+
prefix_phrases: string[];
|
|
682
|
+
suffix_phrases: string[];
|
|
683
|
+
};
|
|
684
|
+
type NativeDenyListMatchData = {
|
|
685
|
+
labels?: string[][];
|
|
686
|
+
label_table?: string[];
|
|
687
|
+
label_indices?: number[][];
|
|
688
|
+
custom_labels?: string[][];
|
|
689
|
+
custom_label_indices?: number[][];
|
|
690
|
+
originals: string[];
|
|
691
|
+
sources?: string[][];
|
|
692
|
+
source_table?: string[];
|
|
693
|
+
source_indices?: number[][];
|
|
694
|
+
filters?: NativeDenyListFilterData;
|
|
695
|
+
};
|
|
696
|
+
type NativeTriggerStrategy = {
|
|
697
|
+
type: "to-next-comma";
|
|
698
|
+
stop_words?: string[];
|
|
699
|
+
max_length?: number;
|
|
700
|
+
} | {
|
|
701
|
+
type: "to-end-of-line";
|
|
702
|
+
} | {
|
|
703
|
+
type: "n-words";
|
|
704
|
+
count: number;
|
|
705
|
+
} | {
|
|
706
|
+
type: "company-id-value";
|
|
707
|
+
} | {
|
|
708
|
+
type: "address";
|
|
709
|
+
max_chars?: number;
|
|
710
|
+
} | {
|
|
711
|
+
type: "match-pattern";
|
|
712
|
+
pattern: string;
|
|
713
|
+
flags?: string;
|
|
714
|
+
};
|
|
715
|
+
type NativeTriggerValidation = {
|
|
716
|
+
type: "starts-uppercase";
|
|
717
|
+
} | {
|
|
718
|
+
type: "min-length";
|
|
719
|
+
min: number;
|
|
720
|
+
} | {
|
|
721
|
+
type: "max-length";
|
|
722
|
+
max: number;
|
|
723
|
+
} | {
|
|
724
|
+
type: "no-digits";
|
|
725
|
+
} | {
|
|
726
|
+
type: "has-digits";
|
|
727
|
+
} | {
|
|
728
|
+
type: "matches-pattern";
|
|
729
|
+
pattern: string;
|
|
730
|
+
flags?: string;
|
|
731
|
+
} | {
|
|
732
|
+
type: "valid-id";
|
|
733
|
+
validator: string;
|
|
734
|
+
};
|
|
735
|
+
type NativeTriggerRule = {
|
|
736
|
+
trigger: string;
|
|
737
|
+
label: string;
|
|
738
|
+
strategy: NativeTriggerStrategy;
|
|
739
|
+
validations: NativeTriggerValidation[];
|
|
740
|
+
include_trigger: boolean;
|
|
741
|
+
};
|
|
742
|
+
type NativeTriggerData = {
|
|
743
|
+
rules: NativeTriggerRule[];
|
|
744
|
+
address_stop_keywords: string[];
|
|
745
|
+
party_position_terms: string[];
|
|
746
|
+
post_nominals: string[];
|
|
747
|
+
sentence_terminal_currency_terms: string[];
|
|
748
|
+
phone_extension_labels: string[];
|
|
749
|
+
number_markers: string[];
|
|
750
|
+
number_labels: string[];
|
|
751
|
+
};
|
|
752
|
+
type NativeLegalFormData = {
|
|
753
|
+
suffixes: string[];
|
|
754
|
+
normalized_boundary_suffixes: string[];
|
|
755
|
+
normalized_in_name_words: string[];
|
|
756
|
+
normalized_suffix_words: string[];
|
|
757
|
+
role_heads: string[];
|
|
758
|
+
sentence_verb_indicators: string[];
|
|
759
|
+
clause_noun_heads: string[];
|
|
760
|
+
connector_prose_heads: string[];
|
|
761
|
+
structural_single_cap_prefixes: string[];
|
|
762
|
+
leading_clause_phrases: string[];
|
|
763
|
+
leading_clause_direct_prefixes: string[];
|
|
764
|
+
connector_words: string[];
|
|
765
|
+
and_connector_words: string[];
|
|
766
|
+
in_name_prepositions: string[];
|
|
767
|
+
company_suffix_words: string[];
|
|
768
|
+
comma_gated_direct_prefixes: string[];
|
|
769
|
+
};
|
|
770
|
+
type NativeDateData = {
|
|
771
|
+
month_names_by_language: DateMonthData;
|
|
772
|
+
year_words_by_language: YearWordData;
|
|
773
|
+
};
|
|
774
|
+
type NativeMonetaryData = MonetaryData;
|
|
775
|
+
type NativeAddressSeedData = AddressSeedData;
|
|
776
|
+
type NativeAddressContextData = AddressContextData;
|
|
777
|
+
type NativeCoreferencePatternData = {
|
|
778
|
+
pattern: string;
|
|
779
|
+
flags: string;
|
|
780
|
+
};
|
|
781
|
+
type NativeCoreferenceData = {
|
|
782
|
+
definition_patterns: NativeCoreferencePatternData[];
|
|
783
|
+
role_stop_terms: string[];
|
|
784
|
+
legal_form_aliases: string[];
|
|
785
|
+
organization_suffixes: string[];
|
|
786
|
+
organization_determiners: string[];
|
|
787
|
+
};
|
|
788
|
+
type NativeNameCorpusData = {
|
|
789
|
+
first_names: string[];
|
|
790
|
+
surnames: string[];
|
|
791
|
+
title_tokens: string[];
|
|
792
|
+
title_abbreviations: string[];
|
|
793
|
+
excluded_words: string[];
|
|
794
|
+
common_words: string[];
|
|
795
|
+
non_western_names: string[];
|
|
796
|
+
excluded_all_caps: string[];
|
|
797
|
+
ja_suffixes: string[];
|
|
798
|
+
arabic_connectors: string[];
|
|
799
|
+
relation_connectors: string[];
|
|
800
|
+
hyphenated_prefixes: string[];
|
|
801
|
+
cjk_non_person_terms: string[];
|
|
802
|
+
cjk_surname_starters: string[];
|
|
803
|
+
organization_terms: string[];
|
|
804
|
+
};
|
|
805
|
+
type NativeNameCorpusMode = "full" | "supplemental";
|
|
806
|
+
type NativeZonePatternData = {
|
|
807
|
+
pattern: string;
|
|
808
|
+
flags: string;
|
|
809
|
+
};
|
|
810
|
+
type NativeZoneSigningClauseData = {
|
|
811
|
+
prefix: string;
|
|
812
|
+
suffix: string;
|
|
813
|
+
prepositions: string[];
|
|
814
|
+
};
|
|
815
|
+
type NativeZoneData = {
|
|
816
|
+
section_heading_patterns: NativeZonePatternData[];
|
|
817
|
+
signing_clauses: NativeZoneSigningClauseData[];
|
|
818
|
+
};
|
|
819
|
+
type NativeGazetteerData = {
|
|
820
|
+
labels: string[];
|
|
821
|
+
is_fuzzy: boolean[];
|
|
822
|
+
};
|
|
823
|
+
type NativeHotwordRule = {
|
|
824
|
+
hotwords: string[];
|
|
825
|
+
target_labels: string[];
|
|
826
|
+
score_adjustment: number;
|
|
827
|
+
reclassify_to?: string;
|
|
828
|
+
proximity_before: number;
|
|
829
|
+
proximity_after: number;
|
|
830
|
+
};
|
|
831
|
+
type NativeHotwordRuleData = {
|
|
832
|
+
rules: NativeHotwordRule[];
|
|
833
|
+
pattern_rule_indices: number[];
|
|
834
|
+
};
|
|
835
|
+
type NativeSignatureData = {
|
|
836
|
+
labels: string[];
|
|
837
|
+
witness_phrases: string[];
|
|
838
|
+
name_particles: string[];
|
|
839
|
+
post_nominal_suffixes: string[];
|
|
840
|
+
organization_suffixes: string[];
|
|
841
|
+
image_stub_prefixes: string[];
|
|
842
|
+
};
|
|
843
|
+
type NativePreparedSearchConfig = {
|
|
844
|
+
regex_patterns: NativeSearchPattern[];
|
|
845
|
+
custom_regex_patterns: NativeSearchPattern[];
|
|
846
|
+
literal_patterns: NativeSearchPattern[];
|
|
847
|
+
regex_options: NativeSearchOptions;
|
|
848
|
+
custom_regex_options: NativeSearchOptions;
|
|
849
|
+
literal_options: NativeSearchOptions;
|
|
850
|
+
literal_patterns_from_deny_list_data?: boolean;
|
|
851
|
+
allowed_labels: string[];
|
|
852
|
+
threshold: number;
|
|
853
|
+
confidence_boost: boolean;
|
|
854
|
+
slices: {
|
|
855
|
+
regex: PatternSlice;
|
|
856
|
+
custom_regex: PatternSlice;
|
|
857
|
+
legal_forms?: PatternSlice;
|
|
858
|
+
triggers?: PatternSlice;
|
|
859
|
+
deny_list: PatternSlice;
|
|
860
|
+
street_types?: PatternSlice;
|
|
861
|
+
gazetteer: PatternSlice;
|
|
862
|
+
countries: PatternSlice;
|
|
863
|
+
hotwords?: PatternSlice;
|
|
864
|
+
};
|
|
865
|
+
regex_meta: NativeRegexMatchMeta[];
|
|
866
|
+
custom_regex_meta: NativeRegexMatchMeta[];
|
|
867
|
+
deny_list_data?: NativeDenyListMatchData;
|
|
868
|
+
false_positive_filters?: NativeDenyListFilterData;
|
|
869
|
+
gazetteer_data?: NativeGazetteerData;
|
|
870
|
+
country_data?: CountryData;
|
|
871
|
+
hotword_data?: NativeHotwordRuleData;
|
|
872
|
+
trigger_data?: NativeTriggerData;
|
|
873
|
+
legal_form_data?: NativeLegalFormData;
|
|
874
|
+
address_seed_data?: NativeAddressSeedData;
|
|
875
|
+
zone_data?: NativeZoneData;
|
|
876
|
+
address_context_data?: NativeAddressContextData;
|
|
877
|
+
coreference_data?: NativeCoreferenceData;
|
|
878
|
+
name_corpus_data?: NativeNameCorpusData;
|
|
879
|
+
signature_data?: NativeSignatureData;
|
|
880
|
+
name_corpus_mode?: NativeNameCorpusMode;
|
|
881
|
+
date_data?: NativeDateData;
|
|
882
|
+
monetary_data?: NativeMonetaryData;
|
|
883
|
+
};
|
|
521
884
|
type GazetteerData = {
|
|
522
885
|
/** Maps local pattern index to entry label. */labels: string[];
|
|
523
886
|
/**
|
|
@@ -546,6 +909,7 @@ type UnifiedSearchInstance = {
|
|
|
546
909
|
denyListData: DenyListData | null;
|
|
547
910
|
gazetteerData: GazetteerData | null;
|
|
548
911
|
countryData: CountryData | null;
|
|
912
|
+
nativeStaticConfig: NativePreparedSearchConfig;
|
|
549
913
|
};
|
|
550
914
|
declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: GazetteerEntry[], ctx?: PipelineContext) => Promise<UnifiedSearchInstance>;
|
|
551
915
|
//#endregion
|
|
@@ -582,7 +946,12 @@ type NameCorpusData = {
|
|
|
582
946
|
* Contains the lowercase, dot-stripped form
|
|
583
947
|
* (e.g., "dr", "smt", "atty"). */
|
|
584
948
|
titleAbbreviations: ReadonlySet<string>;
|
|
585
|
-
excludedWords: ReadonlySet<string>;
|
|
949
|
+
excludedWords: ReadonlySet<string>;
|
|
950
|
+
/** Lowercased common English words. A name chain whose
|
|
951
|
+
* every token is a common word (e.g. "Loan Documents",
|
|
952
|
+
* where "Loan" coincides with a Vietnamese given name)
|
|
953
|
+
* is treated as a common-word phrase, not a person. */
|
|
954
|
+
commonWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
|
|
586
955
|
nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
|
|
587
956
|
excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
|
|
588
957
|
firstNamesList: readonly string[];
|
|
@@ -606,6 +975,9 @@ type PipelineContext = {
|
|
|
606
975
|
search: UnifiedSearchInstance | null;
|
|
607
976
|
searchKey: string;
|
|
608
977
|
searchPromise: Promise<UnifiedSearchInstance> | null;
|
|
978
|
+
nativePipelinePackage: Uint8Array | null;
|
|
979
|
+
nativePipelinePackageKey: string;
|
|
980
|
+
nativePipelinePackagePromise: Promise<Uint8Array> | null;
|
|
609
981
|
nameCorpus: NameCorpusData | null;
|
|
610
982
|
nameCorpusKey: string;
|
|
611
983
|
nameCorpusPromise: Promise<void> | null;
|
|
@@ -615,6 +987,8 @@ type PipelineContext = {
|
|
|
615
987
|
allowListPromise: Promise<ReadonlySet<string>> | null;
|
|
616
988
|
personStopwords: ReadonlySet<string> | null;
|
|
617
989
|
personStopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
990
|
+
definedTermHeads: ReadonlySet<string> | null;
|
|
991
|
+
definedTermHeadsPromise: Promise<ReadonlySet<string>> | null;
|
|
618
992
|
addressStopwords: ReadonlySet<string> | null;
|
|
619
993
|
addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
|
|
620
994
|
firstNameExclusions: ReadonlySet<string> | null;
|
|
@@ -622,6 +996,7 @@ type PipelineContext = {
|
|
|
622
996
|
genericRoles: ReadonlySet<string> | null;
|
|
623
997
|
genericRolesPromise: Promise<ReadonlySet<string>> | null;
|
|
624
998
|
corefPatterns: DefinitionPattern[] | null;
|
|
999
|
+
corefPatternsKey: string;
|
|
625
1000
|
corefPatternsPromise: Promise<DefinitionPattern[]> | null;
|
|
626
1001
|
corefLoadAttempted: boolean;
|
|
627
1002
|
roleStopSet: ReadonlySet<string> | null;
|
|
@@ -689,21 +1064,304 @@ type PipelineOptions = {
|
|
|
689
1064
|
*/
|
|
690
1065
|
declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
|
|
691
1066
|
//#endregion
|
|
1067
|
+
//#region src/native.d.ts
|
|
1068
|
+
type NativeBindingOperatorConfig = {
|
|
1069
|
+
operators?: Record<string, OperatorType>;
|
|
1070
|
+
redactString?: string;
|
|
1071
|
+
};
|
|
1072
|
+
type NativeDiagnosticsBatchCallback = (diagnosticsJson: string) => void;
|
|
1073
|
+
type NativeResultEventCallback = (eventJson: string) => void;
|
|
1074
|
+
type NativeBindingRedactionEntry = {
|
|
1075
|
+
placeholder: string;
|
|
1076
|
+
original: string;
|
|
1077
|
+
};
|
|
1078
|
+
type NativeBindingOperatorEntry = {
|
|
1079
|
+
placeholder: string;
|
|
1080
|
+
operator: OperatorType;
|
|
1081
|
+
};
|
|
1082
|
+
type NativeBindingPipelineEntity = {
|
|
1083
|
+
start: number;
|
|
1084
|
+
end: number;
|
|
1085
|
+
label: string;
|
|
1086
|
+
text: string;
|
|
1087
|
+
score: number;
|
|
1088
|
+
source: string;
|
|
1089
|
+
sourceDetail?: string | null;
|
|
1090
|
+
};
|
|
1091
|
+
type NativeBindingRedactionResult = {
|
|
1092
|
+
redactedText: string;
|
|
1093
|
+
redactionMap: NativeBindingRedactionEntry[];
|
|
1094
|
+
operatorMap: NativeBindingOperatorEntry[];
|
|
1095
|
+
entityCount: number;
|
|
1096
|
+
};
|
|
1097
|
+
type NativeBindingStaticRedactionResult = {
|
|
1098
|
+
resolvedEntities: NativeBindingPipelineEntity[];
|
|
1099
|
+
redaction: NativeBindingRedactionResult;
|
|
1100
|
+
};
|
|
1101
|
+
type NativePreparedSearchBinding = {
|
|
1102
|
+
prepareDiagnosticsJson?: () => string;
|
|
1103
|
+
warmLazyRegex?: () => void;
|
|
1104
|
+
warm_lazy_regex?: () => void;
|
|
1105
|
+
warmLazyRegexDiagnosticsJson?: () => string;
|
|
1106
|
+
warm_lazy_regex_diagnostics_json?: () => string;
|
|
1107
|
+
redactStaticEntities: (fullText: string, operators?: NativeBindingOperatorConfig) => NativeBindingStaticRedactionResult;
|
|
1108
|
+
redactStaticEntitiesJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
1109
|
+
redactStaticEntitiesResultStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onEvent: NativeResultEventCallback) => string;
|
|
1110
|
+
redactStaticEntitiesDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
1111
|
+
redactStaticEntitiesDiagnosticsStreamJson?: (fullText: string, operators: NativeBindingOperatorConfig | undefined, onBatch: NativeDiagnosticsBatchCallback) => string;
|
|
1112
|
+
redactStaticEntitiesSummaryDiagnosticsJson?: (fullText: string, operators?: NativeBindingOperatorConfig) => string;
|
|
1113
|
+
};
|
|
1114
|
+
type NativeAnonymizeBinding = {
|
|
1115
|
+
normalizeForSearch: (text: string) => string;
|
|
1116
|
+
nativePackageVersion: () => string;
|
|
1117
|
+
NativePreparedSearch: {
|
|
1118
|
+
fromConfigJsonBytes: (configJson: Uint8Array) => NativePreparedSearchBinding;
|
|
1119
|
+
fromPreparedPackageBytes: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1120
|
+
fromPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1121
|
+
fromTrustedPreparedPackageBytes?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1122
|
+
fromTrustedPreparedPackageBytesWithoutCache?: (packageBytes: Uint8Array) => NativePreparedSearchBinding;
|
|
1123
|
+
};
|
|
1124
|
+
prepareStaticSearchPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
1125
|
+
prepareStaticSearchCompressedPackageBytes: (configJson: Uint8Array) => Uint8Array;
|
|
1126
|
+
};
|
|
1127
|
+
type NativeOperatorConfig = {
|
|
1128
|
+
operators?: Record<string, OperatorType>;
|
|
1129
|
+
redactString?: string;
|
|
1130
|
+
};
|
|
1131
|
+
type NativePipelineEntity = {
|
|
1132
|
+
start: number;
|
|
1133
|
+
end: number;
|
|
1134
|
+
label: string;
|
|
1135
|
+
text: string;
|
|
1136
|
+
score: number;
|
|
1137
|
+
source: string;
|
|
1138
|
+
sourceDetail?: string;
|
|
1139
|
+
};
|
|
1140
|
+
type NativeRedactionResult = {
|
|
1141
|
+
redactedText: string;
|
|
1142
|
+
redactionMap: Map<string, string>;
|
|
1143
|
+
operatorMap: Map<string, OperatorType>;
|
|
1144
|
+
entityCount: number;
|
|
1145
|
+
};
|
|
1146
|
+
type NativeStaticRedactionResult = {
|
|
1147
|
+
resolvedEntities: NativePipelineEntity[];
|
|
1148
|
+
redaction: NativeRedactionResult;
|
|
1149
|
+
};
|
|
1150
|
+
type NativeSearchPackageOptions = {
|
|
1151
|
+
binding: NativeAnonymizeBinding;
|
|
1152
|
+
config: NativePreparedSearchConfig;
|
|
1153
|
+
compressed?: boolean;
|
|
1154
|
+
};
|
|
1155
|
+
type NativeSearchPackageInput = NativePreparedSearchConfig | string | Uint8Array;
|
|
1156
|
+
type SharedNativeSearchPackageOptions = {
|
|
1157
|
+
binding: NativeAnonymizeBinding;
|
|
1158
|
+
config: NativeSearchPackageInput;
|
|
1159
|
+
compressed?: boolean;
|
|
1160
|
+
};
|
|
1161
|
+
type SharedNativePreparedPackageOptions = {
|
|
1162
|
+
binding: NativeAnonymizeBinding;
|
|
1163
|
+
packageBytes: Uint8Array;
|
|
1164
|
+
};
|
|
1165
|
+
type SharedNativeRedactTextJsonOptions = {
|
|
1166
|
+
binding: NativeAnonymizeBinding;
|
|
1167
|
+
config: NativeSearchPackageInput;
|
|
1168
|
+
fullText: string;
|
|
1169
|
+
operators?: NativeOperatorConfig;
|
|
1170
|
+
};
|
|
1171
|
+
type SharedNativeRedactTextOptions = SharedNativeRedactTextJsonOptions;
|
|
1172
|
+
type SharedNativeDiagnosticsJsonOptions = SharedNativeRedactTextJsonOptions;
|
|
1173
|
+
type SharedNativeRedactTextStreamJsonOptions = SharedNativeRedactTextJsonOptions & {
|
|
1174
|
+
onEvent: NativeResultEventCallback;
|
|
1175
|
+
};
|
|
1176
|
+
type NativeNormalizeOptions = {
|
|
1177
|
+
binding: NativeAnonymizeBinding;
|
|
1178
|
+
text: string;
|
|
1179
|
+
};
|
|
1180
|
+
type NativeAnonymizerFromConfigOptions = {
|
|
1181
|
+
binding: NativeAnonymizeBinding;
|
|
1182
|
+
config: NativePreparedSearchConfig;
|
|
1183
|
+
};
|
|
1184
|
+
type NativeAnonymizerFromPackageOptions = {
|
|
1185
|
+
binding: NativeAnonymizeBinding;
|
|
1186
|
+
packageBytes: Uint8Array;
|
|
1187
|
+
};
|
|
1188
|
+
type NativePipelineFromPackageOptions = NativeAnonymizerFromPackageOptions;
|
|
1189
|
+
type NativeBindingVersionOptions = {
|
|
1190
|
+
binding: NativeAnonymizeBinding;
|
|
1191
|
+
expectedVersion: string;
|
|
1192
|
+
};
|
|
1193
|
+
declare class PreparedNativeAnonymizer {
|
|
1194
|
+
#private;
|
|
1195
|
+
constructor(prepared: NativePreparedSearchBinding);
|
|
1196
|
+
prepareDiagnosticsJson(): string | null;
|
|
1197
|
+
prepare_diagnostics_json(): string | null;
|
|
1198
|
+
warmLazyRegex(): void;
|
|
1199
|
+
warm_lazy_regex(): void;
|
|
1200
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
1201
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
1202
|
+
redactStaticEntities(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1203
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1204
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1205
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1206
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1207
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1208
|
+
redactStaticEntitiesDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1209
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1210
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1211
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1212
|
+
redactStaticEntitiesSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1213
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1214
|
+
}
|
|
1215
|
+
declare class PreparedNativePipeline {
|
|
1216
|
+
#private;
|
|
1217
|
+
constructor(anonymizer: PreparedNativeAnonymizer);
|
|
1218
|
+
prepareDiagnosticsJson(): string | null;
|
|
1219
|
+
prepare_diagnostics_json(): string | null;
|
|
1220
|
+
warmLazyRegex(): void;
|
|
1221
|
+
warm_lazy_regex(): void;
|
|
1222
|
+
warmLazyRegexDiagnosticsJson(): string | null;
|
|
1223
|
+
warm_lazy_regex_diagnostics_json(): string | null;
|
|
1224
|
+
redactText(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1225
|
+
redact_text(fullText: string, operators?: NativeOperatorConfig): NativeStaticRedactionResult;
|
|
1226
|
+
redact_text_json(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1227
|
+
redactTextJson(fullText: string, operators?: NativeOperatorConfig): string;
|
|
1228
|
+
redactTextStreamJson(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1229
|
+
redact_text_stream_json(fullText: string, onEvent: NativeResultEventCallback, operators?: NativeOperatorConfig): string | null;
|
|
1230
|
+
redactTextDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1231
|
+
diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1232
|
+
diagnosticsStreamJson(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1233
|
+
diagnostics_stream_json(fullText: string, onBatch: NativeDiagnosticsBatchCallback, operators?: NativeOperatorConfig): string | null;
|
|
1234
|
+
redactTextSummaryDiagnosticsJson(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1235
|
+
summary_diagnostics_json(fullText: string, operators?: NativeOperatorConfig): string | null;
|
|
1236
|
+
}
|
|
1237
|
+
declare const encodeNativeSearchConfig: (config: NativePreparedSearchConfig) => Uint8Array;
|
|
1238
|
+
declare const encodeNativeSearchConfigInput: (config: NativeSearchPackageInput) => Uint8Array;
|
|
1239
|
+
declare const getNativeBindingVersion: (binding: NativeAnonymizeBinding) => string;
|
|
1240
|
+
declare const native_package_version: (binding: NativeAnonymizeBinding) => string;
|
|
1241
|
+
declare const normalize_for_search: ({
|
|
1242
|
+
binding,
|
|
1243
|
+
text
|
|
1244
|
+
}: NativeNormalizeOptions) => string;
|
|
1245
|
+
declare const assertNativeBindingVersion: ({
|
|
1246
|
+
binding,
|
|
1247
|
+
expectedVersion
|
|
1248
|
+
}: NativeBindingVersionOptions) => void;
|
|
1249
|
+
declare const prepareNativeSearchPackage: ({
|
|
1250
|
+
binding,
|
|
1251
|
+
config,
|
|
1252
|
+
compressed
|
|
1253
|
+
}: NativeSearchPackageOptions) => Uint8Array;
|
|
1254
|
+
declare const prepare_search_package: ({
|
|
1255
|
+
binding,
|
|
1256
|
+
config,
|
|
1257
|
+
compressed
|
|
1258
|
+
}: SharedNativeSearchPackageOptions) => Uint8Array;
|
|
1259
|
+
declare const createNativeAnonymizerFromConfig: ({
|
|
1260
|
+
binding,
|
|
1261
|
+
config
|
|
1262
|
+
}: NativeAnonymizerFromConfigOptions) => PreparedNativeAnonymizer;
|
|
1263
|
+
declare const createNativeAnonymizerFromPackage: ({
|
|
1264
|
+
binding,
|
|
1265
|
+
packageBytes
|
|
1266
|
+
}: NativeAnonymizerFromPackageOptions) => PreparedNativeAnonymizer;
|
|
1267
|
+
declare const load_prepared_package: ({
|
|
1268
|
+
binding,
|
|
1269
|
+
packageBytes
|
|
1270
|
+
}: SharedNativePreparedPackageOptions) => PreparedNativeAnonymizer;
|
|
1271
|
+
declare const redact_text_json: ({
|
|
1272
|
+
binding,
|
|
1273
|
+
config,
|
|
1274
|
+
fullText,
|
|
1275
|
+
operators
|
|
1276
|
+
}: SharedNativeRedactTextJsonOptions) => string;
|
|
1277
|
+
declare const redact_text: ({
|
|
1278
|
+
binding,
|
|
1279
|
+
config,
|
|
1280
|
+
fullText,
|
|
1281
|
+
operators
|
|
1282
|
+
}: SharedNativeRedactTextOptions) => NativeStaticRedactionResult;
|
|
1283
|
+
declare const redact_text_stream_json: ({
|
|
1284
|
+
binding,
|
|
1285
|
+
config,
|
|
1286
|
+
fullText,
|
|
1287
|
+
operators,
|
|
1288
|
+
onEvent
|
|
1289
|
+
}: SharedNativeRedactTextStreamJsonOptions) => string | null;
|
|
1290
|
+
declare const diagnostics_json: ({
|
|
1291
|
+
binding,
|
|
1292
|
+
config,
|
|
1293
|
+
fullText,
|
|
1294
|
+
operators
|
|
1295
|
+
}: SharedNativeDiagnosticsJsonOptions) => string | null;
|
|
1296
|
+
declare const createNativePipelineFromPackage: ({
|
|
1297
|
+
binding,
|
|
1298
|
+
packageBytes
|
|
1299
|
+
}: NativePipelineFromPackageOptions) => PreparedNativePipeline;
|
|
1300
|
+
declare const PreparedSearch: typeof PreparedNativeAnonymizer;
|
|
1301
|
+
type PreparedSearch = PreparedNativeAnonymizer;
|
|
1302
|
+
//#endregion
|
|
1303
|
+
//#region src/native-default-config.d.ts
|
|
1304
|
+
declare const DEFAULT_NATIVE_PIPELINE_CONFIG: PipelineConfig;
|
|
1305
|
+
//#endregion
|
|
1306
|
+
//#region src/native-pipeline.d.ts
|
|
1307
|
+
type NativePipelineUnsupportedFeature = "enableNer";
|
|
1308
|
+
type NativePipelineCompatibility = {
|
|
1309
|
+
status: "supported";
|
|
1310
|
+
} | {
|
|
1311
|
+
status: "unsupported";
|
|
1312
|
+
unsupportedFeatures: NativePipelineUnsupportedFeature[];
|
|
1313
|
+
};
|
|
1314
|
+
type NativePipelineBuildOptions = {
|
|
1315
|
+
binding: NativeAnonymizeBinding;
|
|
1316
|
+
config: PipelineConfig;
|
|
1317
|
+
gazetteerEntries?: GazetteerEntry[];
|
|
1318
|
+
context?: PipelineContext;
|
|
1319
|
+
};
|
|
1320
|
+
type NativePipelinePackageOptions = NativePipelineBuildOptions & {
|
|
1321
|
+
compressed?: boolean;
|
|
1322
|
+
};
|
|
1323
|
+
declare const getNativePipelineCompatibility: (config: PipelineConfig) => NativePipelineCompatibility;
|
|
1324
|
+
declare const assertNativePipelineSupported: (config: PipelineConfig) => void;
|
|
1325
|
+
declare const prepareNativePipelineConfig: ({
|
|
1326
|
+
config,
|
|
1327
|
+
gazetteerEntries,
|
|
1328
|
+
context
|
|
1329
|
+
}: Omit<NativePipelineBuildOptions, "binding">) => Promise<NativePreparedSearchConfig>;
|
|
1330
|
+
declare const prepareNativePipelinePackage: ({
|
|
1331
|
+
binding,
|
|
1332
|
+
config,
|
|
1333
|
+
gazetteerEntries,
|
|
1334
|
+
context,
|
|
1335
|
+
compressed
|
|
1336
|
+
}: NativePipelinePackageOptions) => Promise<Uint8Array>;
|
|
1337
|
+
declare const createNativePipelineFromConfig: ({
|
|
1338
|
+
binding,
|
|
1339
|
+
config,
|
|
1340
|
+
gazetteerEntries,
|
|
1341
|
+
context
|
|
1342
|
+
}: NativePipelineBuildOptions) => Promise<PreparedNativePipeline>;
|
|
1343
|
+
//#endregion
|
|
692
1344
|
//#region src/redact.d.ts
|
|
693
1345
|
/**
|
|
694
1346
|
* Build a stable mapping from entity text to numbered
|
|
695
1347
|
* placeholders. Same real-world value always maps to the
|
|
696
1348
|
* same placeholder (e.g., "Dr. Muller" and "Dr. Muller"
|
|
697
|
-
*
|
|
1349
|
+
* share one person placeholder).
|
|
698
1350
|
*
|
|
699
|
-
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
700
|
-
*
|
|
1351
|
+
* Placeholder format: [LABEL_N] where LABEL is uppercase.
|
|
1352
|
+
* N is allocated per label and skips tokens already present
|
|
1353
|
+
* in reserved text.
|
|
701
1354
|
*
|
|
702
1355
|
* @param _ctx Unused. Kept for signature compatibility;
|
|
703
1356
|
* coref alias links now travel on the entities
|
|
704
1357
|
* themselves (`corefSourceText`).
|
|
705
1358
|
*/
|
|
706
|
-
|
|
1359
|
+
type PlaceholderMapOptions = {
|
|
1360
|
+
reservedText?: string;
|
|
1361
|
+
};
|
|
1362
|
+
declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext, {
|
|
1363
|
+
reservedText
|
|
1364
|
+
}?: PlaceholderMapOptions) => Map<string, string>;
|
|
707
1365
|
/**
|
|
708
1366
|
* Apply redactions to the source text, replacing each
|
|
709
1367
|
* confirmed entity span using the configured operator.
|
|
@@ -775,7 +1433,7 @@ type TriggerPatterns = {
|
|
|
775
1433
|
patterns: string[];
|
|
776
1434
|
rules: TriggerRule[];
|
|
777
1435
|
};
|
|
778
|
-
declare const buildTriggerPatterns: () => Promise<TriggerPatterns>;
|
|
1436
|
+
declare const buildTriggerPatterns: (languages?: readonly string[]) => Promise<TriggerPatterns>;
|
|
779
1437
|
/**
|
|
780
1438
|
* Process trigger matches from the unified search.
|
|
781
1439
|
* Receives all matches; filters to the trigger slice
|
|
@@ -785,20 +1443,6 @@ declare const buildTriggerPatterns: () => Promise<TriggerPatterns>;
|
|
|
785
1443
|
*/
|
|
786
1444
|
declare const processTriggerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, rules: readonly TriggerRule[]) => Entity[];
|
|
787
1445
|
//#endregion
|
|
788
|
-
//#region src/detectors/address-seeds.d.ts
|
|
789
|
-
declare const buildStreetTypePatterns: () => Promise<string[]>;
|
|
790
|
-
/**
|
|
791
|
-
* Process address seeds from the unified search.
|
|
792
|
-
* Receives all matches; filters to the street types
|
|
793
|
-
* slice via sliceStart/sliceEnd. Uses fullText and
|
|
794
|
-
* existingEntities for seed collection, clustering,
|
|
795
|
-
* expansion, and scoring.
|
|
796
|
-
*
|
|
797
|
-
* Runs as a post-processor after all other detectors,
|
|
798
|
-
* using their output as seed sources.
|
|
799
|
-
*/
|
|
800
|
-
declare const processAddressSeeds: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, existingEntities: Entity[]) => Promise<Entity[]>;
|
|
801
|
-
//#endregion
|
|
802
1446
|
//#region src/detectors/gazetteer.d.ts
|
|
803
1447
|
/**
|
|
804
1448
|
* Build TextSearch-compatible patterns from gazetteer
|
|
@@ -830,13 +1474,16 @@ declare const buildGazetteerPatterns: (entries: GazetteerEntry[]) => {
|
|
|
830
1474
|
declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText: string, data: GazetteerData) => Entity[];
|
|
831
1475
|
//#endregion
|
|
832
1476
|
//#region src/detectors/coreference.d.ts
|
|
1477
|
+
type ExtractDefinedTermsOptions = {
|
|
1478
|
+
languages?: readonly string[];
|
|
1479
|
+
};
|
|
833
1480
|
type DefinedTerm = {
|
|
834
1481
|
alias: string;
|
|
835
1482
|
label: string; /** Position of the definition in the source text */
|
|
836
1483
|
definitionStart: number; /** Original entity text the alias refers to */
|
|
837
1484
|
sourceText: string;
|
|
838
1485
|
};
|
|
839
|
-
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext) => Promise<DefinedTerm[]>;
|
|
1486
|
+
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext, options?: ExtractDefinedTermsOptions) => Promise<DefinedTerm[]>;
|
|
840
1487
|
/**
|
|
841
1488
|
* Find all occurrences of defined-term aliases in the
|
|
842
1489
|
* full text. Returns Entity spans for each match.
|
|
@@ -973,21 +1620,6 @@ declare const initAddressComponents: () => Promise<void>;
|
|
|
973
1620
|
*/
|
|
974
1621
|
declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext, fullText?: string) => Entity[];
|
|
975
1622
|
//#endregion
|
|
976
|
-
//#region src/filters/confidence-boost.d.ts
|
|
977
|
-
/**
|
|
978
|
-
* Boost confidence of near-miss NER entities that appear
|
|
979
|
-
* near high-confidence detections (regex, trigger phrase).
|
|
980
|
-
*
|
|
981
|
-
* If an NER entity scored between (threshold - 0.15) and
|
|
982
|
-
* threshold, count how many confirmed entities exist within
|
|
983
|
-
* a 150-char window. Add +0.05 per co-located entity.
|
|
984
|
-
* If the boosted score crosses the threshold, include it.
|
|
985
|
-
*
|
|
986
|
-
* Only mutates score on near-miss entities; high-confidence
|
|
987
|
-
* entities pass through unchanged.
|
|
988
|
-
*/
|
|
989
|
-
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
990
|
-
//#endregion
|
|
991
1623
|
//#region src/filters/hotword-rules.d.ts
|
|
992
1624
|
type HotwordRule = {
|
|
993
1625
|
hotwords: string[];
|
|
@@ -1110,17 +1742,45 @@ declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: st
|
|
|
1110
1742
|
};
|
|
1111
1743
|
//#endregion
|
|
1112
1744
|
//#region src/util/chunker.d.ts
|
|
1745
|
+
/** A chunk paired with its start offset in the source text. */
|
|
1746
|
+
type ChunkSpan = {
|
|
1747
|
+
text: string;
|
|
1748
|
+
offset: number;
|
|
1749
|
+
};
|
|
1750
|
+
/**
|
|
1751
|
+
* Split text into overlapping chunks, each paired with its
|
|
1752
|
+
* exact start offset in the source text.
|
|
1753
|
+
*
|
|
1754
|
+
* Carrying the offset out of the splitter is the robust way to
|
|
1755
|
+
* map chunk-local entity offsets back to document offsets:
|
|
1756
|
+
* downstream code never has to re-locate a chunk by content
|
|
1757
|
+
* search (which mis-locates when boilerplate repeats; see
|
|
1758
|
+
* computeChunkOffsets).
|
|
1759
|
+
*
|
|
1760
|
+
* Character-based splitting (rough token approximation for
|
|
1761
|
+
* GLiNER's ~512 token window); breaks at sentence boundaries
|
|
1762
|
+
* when possible.
|
|
1763
|
+
*/
|
|
1764
|
+
declare const chunkTextWithOffsets: (text: string) => ChunkSpan[];
|
|
1113
1765
|
/**
|
|
1114
|
-
* Split text into overlapping chunks for GLiNER's
|
|
1115
|
-
*
|
|
1116
|
-
*
|
|
1766
|
+
* Split text into overlapping chunks for GLiNER's ~512 token
|
|
1767
|
+
* context window. Character-based splitting (rough token
|
|
1768
|
+
* approximation); breaks at sentence boundaries when possible.
|
|
1117
1769
|
*
|
|
1118
|
-
*
|
|
1770
|
+
* Prefer chunkTextWithOffsets when you also need each chunk's
|
|
1771
|
+
* document offset.
|
|
1119
1772
|
*/
|
|
1120
1773
|
declare const chunkText: (text: string) => string[];
|
|
1121
1774
|
/**
|
|
1122
|
-
* Compute the
|
|
1123
|
-
*
|
|
1775
|
+
* Compute the start offset of each chunk within the original
|
|
1776
|
+
* document text by content search.
|
|
1777
|
+
*
|
|
1778
|
+
* @deprecated Re-locates each chunk with `indexOf`, which can
|
|
1779
|
+
* match the wrong position when identical content repeats in
|
|
1780
|
+
* the document (common in boilerplate-heavy legal text) and
|
|
1781
|
+
* then desyncs every subsequent offset. Use
|
|
1782
|
+
* `chunkTextWithOffsets`, which carries exact offsets out of
|
|
1783
|
+
* the splitter.
|
|
1124
1784
|
*/
|
|
1125
1785
|
declare const computeChunkOffsets: (fullText: string, chunks: string[]) => number[];
|
|
1126
1786
|
/**
|
|
@@ -1179,5 +1839,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
|
|
|
1179
1839
|
*/
|
|
1180
1840
|
declare const normalizeForSearch: (text: string) => string;
|
|
1181
1841
|
//#endregion
|
|
1182
|
-
export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
1842
|
+
export { type AnonymisationOperator, CURRENCY_PATTERN_META, type ChunkSpan, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NativeAnonymizeBinding, type NativeAnonymizerFromConfigOptions, type NativeAnonymizerFromPackageOptions, type NativeBindingVersionOptions, type NativeNormalizeOptions, type NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, type NativePipelineEntity, type NativePipelineFromPackageOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, type NativePreparedSearchBinding, type NativeRedactionResult, type NativeResultEventCallback, type NativeSearchPackageInput, type NativeSearchPackageOptions, type NativeStaticRedactionResult, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, type PreparedSearch as PreparedSearchInstance, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type SharedNativeDiagnosticsJsonOptions, type SharedNativePreparedPackageOptions, type SharedNativeRedactTextJsonOptions, type SharedNativeRedactTextOptions, type SharedNativeRedactTextStreamJsonOptions, type SharedNativeSearchPackageOptions, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, assertNativeBindingVersion, assertNativePipelineSupported, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, chunkTextWithOffsets, classifyZones, computeChunkOffsets, corefKey, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromPackage, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, diagnostics_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, getNativeBindingVersion, getNativePipelineCompatibility, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, load_prepared_package, mergeAndDedup, mergeChunkEntities, native_package_version, normalizeForSearch, normalize_for_search, prepareBatch, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, preparePipelineSearch, prepare_search_package, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, redact_text, redact_text_json, redact_text_stream_json, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
1183
1843
|
//# sourceMappingURL=wasm.d.mts.map
|