@stll/anonymize-wasm 1.4.10 → 1.5.0

package/dist/names-nw-vi.mjs

@@ -0,0 +1,151 @@
+//#region src/data/names-nw-vi.json
+var _comment = "Non-Western name tokens for vi locale";
+var names = [
+	"Anh",
+	"Bao",
+	"Bich",
+	"Bind",
+	"Binh",
+	"Bui",
+	"Cao",
+	"Chanh",
+	"Chau",
+	"Chien",
+	"Chon",
+	"Chuan",
+	"Cuc",
+	"Cuong",
+	"Dang",
+	"Dao",
+	"Dat",
+	"Dau",
+	"Diep",
+	"Dinh",
+	"Doanh",
+	"Dok",
+	"Duc",
+	"Dung",
+	"Duong",
+	"Duyen",
+	"Giang",
+	"Giao",
+	"Hai",
+	"Han",
+	"Hang",
+	"Hanh",
+	"Hao",
+	"Hien",
+	"Hieu",
+	"Hoa",
+	"Hoang",
+	"Hong",
+	"Huong",
+	"Huu",
+	"Huy",
+	"Huynh",
+	"Khang",
+	"Khanh",
+	"Khoa",
+	"Khuyen",
+	"Kie",
+	"Kien",
+	"Kieu",
+	"Kim",
+	"Kiu",
+	"La",
+	"Lam",
+	"Lan",
+	"Lao",
+	"Le",
+	"Lien",
+	"Linh",
+	"Loan",
+	"Loc",
+	"Long",
+	"Luong",
+	"Luu",
+	"Ly",
+	"Mai",
+	"Manh",
+	"Minh",
+	"My",
+	"Nam",
+	"Nga",
+	"Nghia",
+	"Nghiem",
+	"Ngo",
+	"Ngoc",
+	"Nguyen",
+	"Nhan",
+	"Nhat",
+	"Nhi",
+	"Nhu",
+	"Nhung",
+	"Ninh",
+	"Nong",
+	"Oanh",
+	"Pham",
+	"Phan",
+	"Phat",
+	"Phong",
+	"Phu",
+	"Phuc",
+	"Phung",
+	"Phuong",
+	"Quach",
+	"Quan",
+	"Quang",
+	"Quy",
+	"Quyet",
+	"Quynh",
+	"Sang",
+	"Sen",
+	"Sy",
+	"Ta",
+	"Tan",
+	"Thanh",
+	"Thao",
+	"Thi",
+	"Thien",
+	"Thiet",
+	"Thip",
+	"Thu",
+	"Thuan",
+	"Thuc",
+	"Thuy",
+	"Tien",
+	"Tram",
+	"Tran",
+	"Trang",
+	"Trieu",
+	"Trinh",
+	"Trong",
+	"Truc",
+	"Trung",
+	"Truong",
+	"Tuan",
+	"Tuong",
+	"Tuyet",
+	"Ty",
+	"Ung",
+	"Uy",
+	"Uyen",
+	"Vien",
+	"Viet",
+	"Vinh",
+	"Vo",
+	"Vu",
+	"Vuong",
+	"Vy",
+	"Xuan",
+	"Yat",
+	"Yen"
+];
+var names_nw_vi_default = {
+	_comment,
+	names
+};
+//#endregion
+export { _comment, names_nw_vi_default as default, names };
+
+//# sourceMappingURL=names-nw-vi.mjs.map

package/dist/names-nw-vi.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"names-nw-vi.mjs","names":[],"sources":["../../src/data/names-nw-vi.json"],"sourcesContent":[""],"mappings":""}

package/dist/names-nw-zh-latn.mjs

@@ -0,0 +1,197 @@
+//#region src/data/names-nw-zh-latn.json
+var _comment = "Non-Western name tokens for zh-latn locale";
+var names = [
+	"Ang",
+	"AuYeung",
+	"Cao",
+	"Chak",
+	"Cham",
+	"Chan",
+	"Chang",
+	"Chao",
+	"Chau",
+	"Cheak",
+	"Chee",
+	"Chek",
+	"Chen",
+	"Cheong",
+	"Cheung",
+	"Chia",
+	"Chien",
+	"Chik",
+	"Chin",
+	"Ching",
+	"Chiu",
+	"Chng",
+	"Chon",
+	"Chong",
+	"Chow",
+	"Choy",
+	"Chu",
+	"Chua",
+	"Chuan",
+	"Chui",
+	"Chun",
+	"Chung",
+	"Cu",
+	"Cua",
+	"Dai",
+	"Dong",
+	"Dy",
+	"Eah",
+	"Fong",
+	"Foo",
+	"Fuk",
+	"Fung",
+	"Goh",
+	"Guan",
+	"Hak",
+	"Han",
+	"Hang",
+	"Hao",
+	"Heng",
+	"Heung",
+	"Ho",
+	"Hock",
+	"Hoi",
+	"Hong",
+	"Hsu",
+	"Huang",
+	"Huat",
+	"Hui",
+	"Hung",
+	"Jia",
+	"Jiang",
+	"Jun",
+	"Kam",
+	"Kan",
+	"Kee",
+	"Kek",
+	"Keung",
+	"Khoo",
+	"Kian",
+	"Kiat",
+	"Kie",
+	"Kieu",
+	"Kiok",
+	"Kiu",
+	"Ko",
+	"Koh",
+	"Kong",
+	"Koo",
+	"Koon",
+	"Kowk",
+	"Kuk",
+	"Kwan",
+	"Kwok",
+	"Kwong",
+	"La",
+	"Lai",
+	"Lam",
+	"Lau",
+	"Lee",
+	"Leong",
+	"Leung",
+	"Li",
+	"Lim",
+	"Lin",
+	"Ling",
+	"Liu",
+	"Lo",
+	"Lok",
+	"Lone",
+	"Long",
+	"Loo",
+	"Luk",
+	"Lun",
+	"Ma",
+	"Mak",
+	"Mei",
+	"Ming",
+	"Mok",
+	"Mong",
+	"Nai",
+	"Ng",
+	"Ngai",
+	"Ooi",
+	"Oon",
+	"Pai",
+	"Pak",
+	"Pang",
+	"Peh",
+	"Phua",
+	"Ping",
+	"Poe",
+	"Poon",
+	"Pua",
+	"Puah",
+	"Quek",
+	"Ren",
+	"Sen",
+	"Seng",
+	"Seow",
+	"Sham",
+	"Shek",
+	"Shen",
+	"Shum",
+	"Situ",
+	"Siu",
+	"Soh",
+	"Song",
+	"Soon",
+	"Suen",
+	"Suk",
+	"Sy",
+	"Szeto",
+	"Tai",
+	"Tak",
+	"Tam",
+	"Tan",
+	"Tang",
+	"Tay",
+	"Tee",
+	"Teo",
+	"Tham",
+	"Thom",
+	"Tiah",
+	"Ting",
+	"Tiu",
+	"Tong",
+	"Tsai",
+	"Tse",
+	"Tsen",
+	"Tsui",
+	"Tung",
+	"Wai",
+	"Wan",
+	"Wang",
+	"Wee",
+	"Wing",
+	"Wong",
+	"Woo",
+	"Wu",
+	"Wun",
+	"Xie",
+	"Yam",
+	"Yang",
+	"Yap",
+	"Yeoh",
+	"Yeung",
+	"Yi",
+	"Yim",
+	"Ying",
+	"Yip",
+	"Yiu",
+	"Yuen",
+	"Yuk",
+	"Yum",
+	"Zhang"
+];
+var names_nw_zh_latn_default = {
+	_comment,
+	names
+};
+//#endregion
+export { _comment, names_nw_zh_latn_default as default, names };
+
+//# sourceMappingURL=names-nw-zh-latn.mjs.map

package/dist/names-nw-zh-latn.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"names-nw-zh-latn.mjs","names":[],"sources":["../../src/data/names-nw-zh-latn.json"],"sourcesContent":[""],"mappings":""}

package/dist/wasm.d.mts

@@ -392,7 +392,8 @@ type RegexMeta = {
   label: string;
   score: number;
   sourceDetail?: Entity["sourceDetail"]; /** Post-match stdnum validator for confirmation. */
-  validator?: Validator;
+  validator?: Validator; /** Extract the identifier portion when context is part of the regex span. */
+  validatorInput?: (text: string) => string;
 };
 /** Flat pattern array for text-search. */
 declare const REGEX_PATTERNS: readonly string[];
@@ -582,7 +583,12 @@ type NameCorpusData = {
    *  Contains the lowercase, dot-stripped form
    *  (e.g., "dr", "smt", "atty"). */
   titleAbbreviations: ReadonlySet<string>;
-  excludedWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
+  excludedWords: ReadonlySet<string>;
+  /** Lowercased common English words. A name chain whose
+   *  every token is a common word (e.g. "Loan Documents",
+   *  where "Loan" coincides with a Vietnamese given name)
+   *  is treated as a common-word phrase, not a person. */
+  commonWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
   nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
   excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
   firstNamesList: readonly string[];
@@ -1110,17 +1116,45 @@ declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: st
 };
 //#endregion
 //#region src/util/chunker.d.ts
+/** A chunk paired with its start offset in the source text. */
+type ChunkSpan = {
+  text: string;
+  offset: number;
+};
 /**
- * Split text into overlapping chunks for GLiNER's
- * ~512 token context window. Character-based splitting
- * (rough approximation of token limits).
+ * Split text into overlapping chunks, each paired with its
+ * exact start offset in the source text.
  *
- * Tries to break at sentence boundaries when possible.
+ * Carrying the offset out of the splitter is the robust way to
+ * map chunk-local entity offsets back to document offsets:
+ * downstream code never has to re-locate a chunk by content
+ * search (which mis-locates when boilerplate repeats; see
+ * computeChunkOffsets).
+ *
+ * Character-based splitting (rough token approximation for
+ * GLiNER's ~512 token window); breaks at sentence boundaries
+ * when possible.
+ */
+declare const chunkTextWithOffsets: (text: string) => ChunkSpan[];
+/**
+ * Split text into overlapping chunks for GLiNER's ~512 token
+ * context window. Character-based splitting (rough token
+ * approximation); breaks at sentence boundaries when possible.
+ *
+ * Prefer chunkTextWithOffsets when you also need each chunk's
+ * document offset.
  */
 declare const chunkText: (text: string) => string[];
 /**
- * Compute the byte offset of each chunk within the
- * original document text.
+ * Compute the start offset of each chunk within the original
+ * document text by content search.
+ *
+ * @deprecated Re-locates each chunk with `indexOf`, which can
+ *   match the wrong position when identical content repeats in
+ *   the document (common in boilerplate-heavy legal text) and
+ *   then desyncs every subsequent offset. Use
+ *   `chunkTextWithOffsets`, which carries exact offsets out of
+ *   the splitter.
  */
 declare const computeChunkOffsets: (fullText: string, chunks: string[]) => number[];
 /**
@@ -1179,5 +1213,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
  */
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
-export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
+export { type AnonymisationOperator, CURRENCY_PATTERN_META, type ChunkSpan, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, chunkTextWithOffsets, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
 //# sourceMappingURL=wasm.d.mts.map