@stll/anonymize-wasm 1.4.11 → 1.5.0

package/dist/address-stopwords.mjs

@@ -6,6 +6,7 @@ var address_stopwords_default = {
 		"agenda",
 		"agent",
 		"amount",
+		"basic",
 		"brent",
 		"central",
 		"centro",
@@ -32,6 +33,7 @@ var address_stopwords_default = {
 		"price",
 		"principal",
 		"rate",
+		"rent",
 		"registro",
 		"schedule",
 		"se",

package/dist/allow-list.mjs

@@ -16,6 +16,7 @@ var allow_list_default = {
 		"benefit",
 		"bic",
 		"board",
+		"bond",
 		"bonus",
 		"branch",
 		"business",

package/dist/constants.mjs

@@ -78,6 +78,7 @@ const DEFAULT_ENTITY_LABELS = [
 	"registration number",
 	"credit card number",
 	"passport number",
+	"crypto",
 	"monetary amount",
 	"land parcel",
 	"misc"

package/dist/constants.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"constants.mjs","names":[],"sources":["../../src/constants.ts"],"sourcesContent":["/**\n * Runtime-free constants for the anonymization pipeline.\n *\n * This module is the SSR-safe / browser-safe entrypoint:\n * importing it must not pull in `@stll/text-search`,\n * `@stll/anonymize-wasm`, or any other runtime-bearing\n * module. Consumers that only need the static label list,\n * detection-source identifiers, or operator names import\n * from `@stll/anonymize/constants` (or\n * `@stll/anonymize-wasm/constants`) without paying the\n * wasm / regex-set startup cost.\n *\n * `types.ts` re-exports these for back-compat, so existing\n * `import { DEFAULT_ENTITY_LABELS } from \"@stll/anonymize\"`\n * call sites keep working.\n */\n\n/**\n * Source of a detected entity span.\n * Ordered by detection layer in the pipeline.\n */\nexport const DETECTION_SOURCES = {\n  TRIGGER: \"trigger\",\n  REGEX: \"regex\",\n  DENY_LIST: \"deny-list\",\n  LEGAL_FORM: \"legal-form\",\n  GAZETTEER: \"gazetteer\",\n  COUNTRY: \"country\",\n  NER: \"ner\",\n  COREFERENCE: \"coreference\",\n} as const;\n\nexport type DetectionSource =\n  (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];\n\n/**\n * Priority levels for detection sources.\n * Higher = more structurally reliable. Used during\n * overlap resolution so deterministic detectors beat\n * probabilistic ones regardless of raw score.\n */\nexport const DETECTOR_PRIORITY = {\n  [DETECTION_SOURCES.GAZETTEER]: 5,\n  [DETECTION_SOURCES.TRIGGER]: 4,\n  [DETECTION_SOURCES.LEGAL_FORM]: 3,\n  [DETECTION_SOURCES.REGEX]: 3,\n  [DETECTION_SOURCES.COUNTRY]: 3,\n  [DETECTION_SOURCES.DENY_LIST]: 2,\n  [DETECTION_SOURCES.COREFERENCE]: 2,\n  [DETECTION_SOURCES.NER]: 1,\n} as const satisfies Record<DetectionSource, number>;\n\n/**\n * Anonymization operator types. Each operator defines\n * how a confirmed entity is replaced in the output.\n */\nexport const OPERATOR_TYPES = [\"replace\", \"redact\"] as const;\n\nexport type OperatorType = (typeof OPERATOR_TYPES)[number];\n\n/**\n * Canonical entity labels used across the pipeline.\n * NER models may use different native labels; the bench\n * NER wrapper maps model output to these canonical names.\n *\n * These labels are ephemeral: entities are regenerated on\n * every pipeline run and never persisted to the database.\n * Renaming a label here requires no migration.\n */\nexport const DEFAULT_ENTITY_LABELS = [\n  \"person\",\n  \"organization\",\n  \"phone number\",\n  \"address\",\n  \"country\",\n  \"email address\",\n  \"date\",\n  \"date of birth\",\n  \"bank account number\",\n  \"iban\",\n  \"tax identification number\",\n  \"identity card number\",\n  \"birth number\",\n  \"national identification number\",\n  \"social security number\",\n  \"registration number\",\n  \"credit card number\",\n  \"passport number\",\n  \"monetary amount\",\n  \"land parcel\",\n  \"misc\",\n] as const;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAqBA,MAAa,oBAAoB;CAC/B,SAAS;CACT,OAAO;CACP,WAAW;CACX,YAAY;CACZ,WAAW;CACX,SAAS;CACT,KAAK;CACL,aAAa;AACf;;;;;;;AAWA,MAAa,oBAAoB;EAC9B,kBAAkB,YAAY;EAC9B,kBAAkB,UAAU;EAC5B,kBAAkB,aAAa;EAC/B,kBAAkB,QAAQ;EAC1B,kBAAkB,UAAU;EAC5B,kBAAkB,YAAY;EAC9B,kBAAkB,cAAc;EAChC,kBAAkB,MAAM;AAC3B;;;;;AAMA,MAAa,iBAAiB,CAAC,WAAW,QAAQ;;;;;;;;;;AAalD,MAAa,wBAAwB;CACnC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}
+{"version":3,"file":"constants.mjs","names":[],"sources":["../../src/constants.ts"],"sourcesContent":["/**\n * Runtime-free constants for the anonymization pipeline.\n *\n * This module is the SSR-safe / browser-safe entrypoint:\n * importing it must not pull in `@stll/text-search`,\n * `@stll/anonymize-wasm`, or any other runtime-bearing\n * module. Consumers that only need the static label list,\n * detection-source identifiers, or operator names import\n * from `@stll/anonymize/constants` (or\n * `@stll/anonymize-wasm/constants`) without paying the\n * wasm / regex-set startup cost.\n *\n * `types.ts` re-exports these for back-compat, so existing\n * `import { DEFAULT_ENTITY_LABELS } from \"@stll/anonymize\"`\n * call sites keep working.\n */\n\n/**\n * Source of a detected entity span.\n * Ordered by detection layer in the pipeline.\n */\nexport const DETECTION_SOURCES = {\n  TRIGGER: \"trigger\",\n  REGEX: \"regex\",\n  DENY_LIST: \"deny-list\",\n  LEGAL_FORM: \"legal-form\",\n  GAZETTEER: \"gazetteer\",\n  COUNTRY: \"country\",\n  NER: \"ner\",\n  COREFERENCE: \"coreference\",\n} as const;\n\nexport type DetectionSource =\n  (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];\n\n/**\n * Priority levels for detection sources.\n * Higher = more structurally reliable. Used during\n * overlap resolution so deterministic detectors beat\n * probabilistic ones regardless of raw score.\n */\nexport const DETECTOR_PRIORITY = {\n  [DETECTION_SOURCES.GAZETTEER]: 5,\n  [DETECTION_SOURCES.TRIGGER]: 4,\n  [DETECTION_SOURCES.LEGAL_FORM]: 3,\n  [DETECTION_SOURCES.REGEX]: 3,\n  [DETECTION_SOURCES.COUNTRY]: 3,\n  [DETECTION_SOURCES.DENY_LIST]: 2,\n  [DETECTION_SOURCES.COREFERENCE]: 2,\n  [DETECTION_SOURCES.NER]: 1,\n} as const satisfies Record<DetectionSource, number>;\n\n/**\n * Anonymization operator types. Each operator defines\n * how a confirmed entity is replaced in the output.\n */\nexport const OPERATOR_TYPES = [\"replace\", \"redact\"] as const;\n\nexport type OperatorType = (typeof OPERATOR_TYPES)[number];\n\n/**\n * Canonical entity labels used across the pipeline.\n * NER models may use different native labels; the bench\n * NER wrapper maps model output to these canonical names.\n *\n * These labels are ephemeral: entities are regenerated on\n * every pipeline run and never persisted to the database.\n * Renaming a label here requires no migration.\n */\nexport const DEFAULT_ENTITY_LABELS = [\n  \"person\",\n  \"organization\",\n  \"phone number\",\n  \"address\",\n  \"country\",\n  \"email address\",\n  \"date\",\n  \"date of birth\",\n  \"bank account number\",\n  \"iban\",\n  \"tax identification number\",\n  \"identity card number\",\n  \"birth number\",\n  \"national identification number\",\n  \"social security number\",\n  \"registration number\",\n  \"credit card number\",\n  \"passport number\",\n  \"crypto\",\n  \"monetary amount\",\n  \"land parcel\",\n  \"misc\",\n] as const;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAqBA,MAAa,oBAAoB;CAC/B,SAAS;CACT,OAAO;CACP,WAAW;CACX,YAAY;CACZ,WAAW;CACX,SAAS;CACT,KAAK;CACL,aAAa;AACf;;;;;;;AAWA,MAAa,oBAAoB;EAC9B,kBAAkB,YAAY;EAC9B,kBAAkB,UAAU;EAC5B,kBAAkB,aAAa;EAC/B,kBAAkB,QAAQ;EAC1B,kBAAkB,UAAU;EAC5B,kBAAkB,YAAY;EAC9B,kBAAkB,cAAc;EAChC,kBAAkB,MAAM;AAC3B;;;;;AAMA,MAAa,iBAAiB,CAAC,WAAW,QAAQ;;;;;;;;;;AAalD,MAAa,wBAAwB;CACnC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/constants2.d.mts

@@ -61,7 +61,7 @@ type OperatorType = (typeof OPERATOR_TYPES)[number];
  * every pipeline run and never persisted to the database.
  * Renaming a label here requires no migration.
  */
-declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "country", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "birth number", "national identification number", "social security number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel", "misc"];
+declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "country", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "birth number", "national identification number", "social security number", "registration number", "credit card number", "passport number", "crypto", "monetary amount", "land parcel", "misc"];
 //#endregion
 export { OPERATOR_TYPES as a, DetectionSource as i, DETECTION_SOURCES as n, OperatorType as o, DETECTOR_PRIORITY as r, DEFAULT_ENTITY_LABELS as t };
 //# sourceMappingURL=constants2.d.mts.map

package/dist/wasm.d.mts

@@ -392,7 +392,8 @@ type RegexMeta = {
   label: string;
   score: number;
   sourceDetail?: Entity["sourceDetail"]; /** Post-match stdnum validator for confirmation. */
-  validator?: Validator;
+  validator?: Validator; /** Extract the identifier portion when context is part of the regex span. */
+  validatorInput?: (text: string) => string;
 };
 /** Flat pattern array for text-search. */
 declare const REGEX_PATTERNS: readonly string[];
@@ -582,7 +583,12 @@ type NameCorpusData = {
    *  Contains the lowercase, dot-stripped form
    *  (e.g., "dr", "smt", "atty"). */
   titleAbbreviations: ReadonlySet<string>;
-  excludedWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
+  excludedWords: ReadonlySet<string>;
+  /** Lowercased common English words. A name chain whose
+   *  every token is a common word (e.g. "Loan Documents",
+   *  where "Loan" coincides with a Vietnamese given name)
+   *  is treated as a common-word phrase, not a person. */
+  commonWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
   nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
   excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
   firstNamesList: readonly string[];
@@ -1110,17 +1116,45 @@ declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: st
 };
 //#endregion
 //#region src/util/chunker.d.ts
+/** A chunk paired with its start offset in the source text. */
+type ChunkSpan = {
+  text: string;
+  offset: number;
+};
 /**
- * Split text into overlapping chunks for GLiNER's
- * ~512 token context window. Character-based splitting
- * (rough approximation of token limits).
+ * Split text into overlapping chunks, each paired with its
+ * exact start offset in the source text.
  *
- * Tries to break at sentence boundaries when possible.
+ * Carrying the offset out of the splitter is the robust way to
+ * map chunk-local entity offsets back to document offsets:
+ * downstream code never has to re-locate a chunk by content
+ * search (which mis-locates when boilerplate repeats; see
+ * computeChunkOffsets).
+ *
+ * Character-based splitting (rough token approximation for
+ * GLiNER's ~512 token window); breaks at sentence boundaries
+ * when possible.
+ */
+declare const chunkTextWithOffsets: (text: string) => ChunkSpan[];
+/**
+ * Split text into overlapping chunks for GLiNER's ~512 token
+ * context window. Character-based splitting (rough token
+ * approximation); breaks at sentence boundaries when possible.
+ *
+ * Prefer chunkTextWithOffsets when you also need each chunk's
+ * document offset.
  */
 declare const chunkText: (text: string) => string[];
 /**
- * Compute the byte offset of each chunk within the
- * original document text.
+ * Compute the start offset of each chunk within the original
+ * document text by content search.
+ *
+ * @deprecated Re-locates each chunk with `indexOf`, which can
+ *   match the wrong position when identical content repeats in
+ *   the document (common in boilerplate-heavy legal text) and
+ *   then desyncs every subsequent offset. Use
+ *   `chunkTextWithOffsets`, which carries exact offsets out of
+ *   the splitter.
  */
 declare const computeChunkOffsets: (fullText: string, chunks: string[]) => number[];
 /**
@@ -1179,5 +1213,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
  */
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
-export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
+export { type AnonymisationOperator, CURRENCY_PATTERN_META, type ChunkSpan, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, chunkTextWithOffsets, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
 //# sourceMappingURL=wasm.d.mts.map