@stll/anonymize 0.0.8 → 1.0.0

package/dist/{index.d.ts → index.d.mts}

@@ -1,5 +1,5 @@
-import { Validator } from "@stll/stdnum";
 import { Match, PatternEntry, TextSearch } from "@stll/text-search";
+import { Validator } from "@stll/stdnum";
 import { Tokenizer } from "@huggingface/tokenizers";
 
 //#region src/types.d.ts
@@ -143,9 +143,7 @@ declare const OPERATOR_TYPES: readonly ["replace", "redact"];
 type OperatorType = (typeof OPERATOR_TYPES)[number];
 /** Per-label operator selection. Key is the entity label. */
 type OperatorConfig = {
-  /** Operator per label. Missing labels default to "replace". */
-  operators: Record<string, OperatorType>;
-  /** Custom replacement string for the redact operator. */
+  /** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
   redactString: string;
 };
 /** Whether an operator produces a reversible redaction entry. */
@@ -168,8 +166,7 @@ type RedactionResult = {
    * Maps placeholder to original text. Only populated for
    * reversible operators (replace). Empty for redact.
    */
-  redactionMap: Map<string, string>;
-  /** Maps placeholder to the operator that produced it. */
+  redactionMap: Map<string, string>; /** Maps placeholder to the operator that produced it. */
   operatorMap: Map<string, OperatorType>;
   entityCount: number;
 };
@@ -209,8 +206,7 @@ declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone
 //#region src/detectors/regex.d.ts
 type RegexMeta = {
   label: string;
-  score: number;
-  /** Post-match stdnum validator for confirmation. */
+  score: number; /** Post-match stdnum validator for confirmation. */
   validator?: Validator;
 };
 /** Flat pattern array for text-search. */
@@ -269,10 +265,8 @@ type DenyListData = {
    * appears in multiple dictionaries (e.g., "Denver"
    * is both a person name and a city name).
    */
-  labels: string[][];
-  /** Maps pattern index → original pattern text. */
-  originals: string[];
-  /** Maps pattern index → source types (plural). */
+  labels: string[][]; /** Maps pattern index → original pattern text. */
+  originals: string[]; /** Maps pattern index → source types (plural). */
   sources: PatternSource[][];
 };
 /**
@@ -317,8 +311,7 @@ type PatternSlice = {
   end: number;
 };
 type GazetteerData = {
-  /** Maps local pattern index to entry label. */
-  labels: string[];
+  /** Maps local pattern index to entry label. */labels: string[];
   /**
    * Whether each pattern is fuzzy (distance > 0).
    * Used by the post-processor to assign scores.
@@ -326,9 +319,7 @@ type GazetteerData = {
   isFuzzy: boolean[];
 };
 type UnifiedSearchInstance = {
-  /** Regex + triggers + legal-forms. */
-  tsRegex: TextSearch;
-  /** Deny-list + street-types + gazetteer. */
+  /** Regex + triggers + legal-forms. */tsRegex: TextSearch; /** Deny-list + street-types + gazetteer. */
   tsLiterals: TextSearch;
   slices: {
     regex: PatternSlice;
@@ -369,8 +360,7 @@ type NameCorpusData = {
   firstNames: ReadonlySet<string>;
   surnames: ReadonlySet<string>;
   titleTokens: ReadonlySet<string>;
-  excludedWords: ReadonlySet<string>;
-  /** Raw arrays exposed for deny-list AC integration. */
+  excludedWords: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
   firstNamesList: readonly string[];
   surnamesList: readonly string[];
   titlesList: readonly string[];
@@ -397,8 +387,7 @@ type PipelineContext = {
   allowList: ReadonlySet<string> | null;
   allowListPromise: Promise<ReadonlySet<string>> | null;
   personStopwords: ReadonlySet<string> | null;
-  personStopwordsPromise: Promise<ReadonlySet<string>> | null;
-  /** First-name exclusions for stopword filtering. */
+  personStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
   firstNameExclusions: ReadonlySet<string> | null;
   firstNameExclusionCorpusLen: number;
   genericRoles: ReadonlySet<string> | null;
@@ -425,11 +414,6 @@ type PipelineContext = {
 };
 /** Create a fresh, empty pipeline context. */
 declare const createPipelineContext: () => PipelineContext;
-/**
- * Module-level default context. Used when callers
- * don't provide an explicit context, preserving full
- * backward compatibility with the existing API.
- */
 //#endregion
 //#region src/pipeline.d.ts
 /** Strip leading/trailing whitespace and punctuation. */
@@ -613,10 +597,8 @@ declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number,
 //#region src/detectors/coreference.d.ts
 type DefinedTerm = {
   alias: string;
-  label: string;
-  /** Position of the definition in the source text */
-  definitionStart: number;
-  /** Original entity text the alias refers to */
+  label: string; /** Position of the definition in the source text */
+  definitionStart: number; /** Original entity text the alias refers to */
   sourceText: string;
 };
 declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext) => Promise<DefinedTerm[]>;
@@ -647,7 +629,9 @@ declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entit
 //#endregion
 //#region src/detectors/names.d.ts
 /**
- * Load name corpus data from JSON config files.
+ * Load name corpus data from per-language dictionary
+ * files and legacy config files. Merges all sources.
+ *
  * Safe to call multiple times; only loads once per
  * context. Must be called before detectNameCorpus or
  * the getNameCorpus*() accessors are used.
@@ -673,8 +657,7 @@ declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext) => Ent
 //#endregion
 //#region src/unified-search.d.ts
 type UnifiedResult = {
-  /** All matches from both instances combined. */
-  regexMatches: Match[];
+  /** All matches from both instances combined. */regexMatches: Match[];
   literalMatches: Match[];
 };
 declare const runUnifiedSearch: (instance: UnifiedSearchInstance, fullText: string) => UnifiedResult;
@@ -740,7 +723,6 @@ declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext)
  * entities pass through unchanged.
  */
 declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
-/** Ensure preposition data is loaded. */
 //#endregion
 //#region src/filters/hotword-rules.d.ts
 type HotwordRule = {
@@ -849,8 +831,6 @@ declare const decodeTokenSpans: (batchSize: number, numWords: number, numEntitie
 //#region src/gliner/processor.d.ts
 /** Tokenize text into words with character offsets. */
 declare const tokenizeText: (text: string) => [words: string[], starts: number[], ends: number[]];
-/** Pad a 2D or 3D array to uniform inner length. */
-
 /** Prepare a complete batch for ONNX inference. */
 declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: string[], maxWidth: number) => {
   inputsIds: number[][];
@@ -936,4 +916,4 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
 export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
-//# sourceMappingURL=index.d.ts.map
+//# sourceMappingURL=index.d.mts.map

package/dist/{index.js → index.mjs}

@@ -1,7 +1,20 @@
+import { createRequire } from "node:module";
+import { TextSearch } from "@stll/text-search";
 import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
 import { toRegex } from "@stll/stdnum/patterns";
-import charGroupsJson from "@stll/anonymize-data/config/char-groups.json";
-import { TextSearch } from "@stll/text-search";
+//#region \0rolldown/runtime.js
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+//#endregion
+//#region src/search-engine.ts
+let _TextSearch;
+const initTextSearch = (ctor) => {
+	_TextSearch = ctor;
+};
+const getTextSearch = () => {
+	if (!_TextSearch) throw new Error("TextSearch not initialized. Import from @stll/anonymize or @stll/anonymize-wasm, not from internal modules.");
+	return _TextSearch;
+};
+//#endregion
 //#region src/types.ts
 /**
 * Source of a detected entity span.
@@ -566,7 +579,37 @@ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstN
 const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
 const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
 /**
-* Load name corpus data from JSON config files.
+* Languages with per-language first/surname
+* dictionaries in @stll/anonymize-data.
+*/
+const NAME_LANGUAGES = [
+	"cs",
+	"sk",
+	"de",
+	"pl",
+	"hu",
+	"ro",
+	"fr",
+	"es",
+	"it",
+	"en",
+	"sv"
+];
+/**
+* Try importing a JSON module; return empty array
+* if not found.
+*/
+const tryImportArray = async (path) => {
+	try {
+		return (await import(path)).default;
+	} catch {
+		return [];
+	}
+};
+/**
+* Load name corpus data from per-language dictionary
+* files and legacy config files. Merges all sources.
+*
 * Safe to call multiple times; only loads once per
 * context. Must be called before detectNameCorpus or
 * the getNameCorpus*() accessors are used.
@@ -575,23 +618,40 @@ const initNameCorpus = (ctx = defaultContext) => {
 	if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
 	const promise = (async () => {
 		try {
-			const [firstMod, surnameMod, titleMod, exclusionMod] = await Promise.all([
+			const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
 				import("@stll/anonymize-data/config/names-first.json"),
 				import("@stll/anonymize-data/config/names-surnames.json"),
 				import("@stll/anonymize-data/config/names-title-tokens.json"),
 				import("@stll/anonymize-data/config/names-exclusions.json")
 			]);
-			const firstNames = firstMod.default.names;
-			const surnames = surnameMod.default.names;
+			const firstImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/first/${lang}.json`));
+			const surnameImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/surnames/${lang}.json`));
+			const [firstResults, surnameResults] = await Promise.all([Promise.all(firstImports), Promise.all(surnameImports)]);
+			const firstNames = [...legacyFirstMod.default.names];
+			for (const names of firstResults) for (const name of names) firstNames.push(name);
+			const surnames = [...legacySurnameMod.default.names];
+			for (const names of surnameResults) for (const name of names) surnames.push(name);
+			const dedup = (arr) => {
+				const seen = /* @__PURE__ */ new Set();
+				const result = [];
+				for (const item of arr) {
+					if (seen.has(item)) continue;
+					seen.add(item);
+					result.push(item);
+				}
+				return result;
+			};
+			const dedupFirst = dedup(firstNames);
+			const dedupSurnames = dedup(surnames);
 			const titles = titleMod.default.tokens;
 			const exclusions = exclusionMod.default.words;
 			ctx.nameCorpus = {
-				firstNames: Object.freeze(new Set(firstNames)),
-				surnames: Object.freeze(new Set(surnames)),
+				firstNames: Object.freeze(new Set(dedupFirst)),
+				surnames: Object.freeze(new Set(dedupSurnames)),
 				titleTokens: Object.freeze(new Set(titles)),
 				excludedWords: Object.freeze(new Set(exclusions)),
-				firstNamesList: Object.freeze(firstNames),
-				surnamesList: Object.freeze(surnames),
+				firstNamesList: Object.freeze(dedupFirst),
+				surnamesList: Object.freeze(dedupSurnames),
 				titlesList: Object.freeze(titles),
 				excludedList: Object.freeze(exclusions)
 			};
@@ -999,7 +1059,12 @@ const POST_NOMINALS = [
 /** Chars that need escaping inside a regex char class. */
 const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
 const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
-const loadConfig = () => charGroupsJson;
+let cached;
+const loadConfig = () => {
+	if (cached) return cached;
+	cached = __require("@stll/anonymize-data/config/char-groups.json");
+	return cached;
+};
 /**
 * Get the raw characters for a named group.
 * Throws if the group does not exist.
@@ -1195,7 +1260,7 @@ const CZ_BIRTH_NUMBER = {
 	validator: cz.rc
 };
 const DATE_NUMERIC = {
-	pattern: "\\b(?:\\d{1,2}[./]\\d{1,2}[./]\\d{2,4}|\\d{4}-\\d{2}-\\d{2})\\b",
+	pattern: "\\b(?:\\d{1,2}[./]\\d{1,2}[./]\\d{2,4}|\\d{4}-\\d{2}-\\d{2}|\\d{4}\\.\\d{2}\\.\\d{2})\\b",
 	label: "date",
 	score: 1
 };
@@ -1281,6 +1346,11 @@ const ALL_REGEX_DEFS = [
 		label: "url",
 		score: .9
 	},
+	{
+		pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
+		label: "date",
+		score: .9
+	},
 	...STDNUM_ENTRIES
 ];
 /** Flat pattern array for text-search. */
@@ -2264,6 +2334,7 @@ const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
 const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
 const HAS_DIGIT_RE = /\d/;
 const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
+const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
 const MAX_ENTITY_LENGTH = {
 	organization: 80,
 	person: 60
@@ -2315,8 +2386,8 @@ const filterFalsePositives = (entities, ctx = defaultContext) => {
 		if (STANDALONE_YEAR_RE.test(trimmed)) continue;
 		if (entity.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
 		if ((entity.label === "person" || entity.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
-		if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
-		if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
+		if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
+		if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
 		filtered.push(entity);
 	}
 	return filtered;
@@ -3589,7 +3660,7 @@ const loadRules = async () => {
 			mapping.push(ruleIdx);
 		}
 	}
-	const builtSearch = patterns.length > 0 ? new TextSearch(patterns, {
+	const builtSearch = patterns.length > 0 ? new (getTextSearch())(patterns, {
 		overlapStrategy: "all",
 		caseInsensitive: true,
 		wholeWords: true
@@ -3981,24 +4052,6 @@ const enforceBoundaryConsistency = (entities, fullText) => {
 };
 //#endregion
 //#region src/build-unified-search.ts
-/**
-* Build the unified search instances from all
-* detector pattern sources.
-*
-* Two TextSearch instances (not one) to avoid
-* 200K per-pattern object allocations:
-* 1. regex + triggers + legal-forms (mixed, ~140
-*    patterns, caseInsensitive for trigger AC)
-* 2. deny-list + street-types + gazetteer
-*    (caseInsensitive, overlap "all";
-*    deny-list/street-type use per-pattern
-*    wholeWords: true; gazetteer exact use
-*    wholeWords: false; gazetteer fuzzy use
-*    distance: 2 via @stll/fuzzy-search)
-*
-* All patterns are PatternEntry objects with
-* per-pattern literal/wholeWords settings.
-*/
 const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
 	const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
 		buildLegalFormPatterns(),
@@ -4044,11 +4097,12 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
 		literal: true,
 		caseInsensitive: true
 	}));
-	const tsRegex = new TextSearch([
+	const regexAllPatterns = [
 		...allRegex,
 		...legalForms,
 		...triggerEntries
-	]);
+	];
+	const tsRegex = new (getTextSearch())(regexAllPatterns);
 	offset = 0;
 	const denyListOriginals = denyListData?.originals ?? [];
 	const denyListSlice = {
@@ -4079,10 +4133,10 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
 	];
 	return {
 		tsRegex,
-		tsLiterals: literalAllPatterns.length > 0 ? new TextSearch(literalAllPatterns, {
+		tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
 			caseInsensitive: true,
 			overlapStrategy: "all"
-		}) : new TextSearch([]),
+		}) : new (getTextSearch())([]),
 		slices: {
 			regex: regexSlice,
 			legalForms: legalFormsSlice,
@@ -5050,6 +5104,9 @@ const levenshtein = (rawA, rawB) => {
 	return row[aLen] ?? 0;
 };
 //#endregion
+//#region src/index.ts
+initTextSearch(TextSearch);
+//#endregion
 export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
 
-//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.mjs.map