@stll/anonymize 0.0.9 → 1.0.0

package/dist/{index.js → index.mjs}

@@ -1,6 +1,20 @@
+import { createRequire } from "node:module";
+import { TextSearch } from "@stll/text-search";
 import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
 import { toRegex } from "@stll/stdnum/patterns";
-import { TextSearch } from "@stll/text-search";
+//#region \0rolldown/runtime.js
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+//#endregion
+//#region src/search-engine.ts
+let _TextSearch;
+const initTextSearch = (ctor) => {
+	_TextSearch = ctor;
+};
+const getTextSearch = () => {
+	if (!_TextSearch) throw new Error("TextSearch not initialized. Import from @stll/anonymize or @stll/anonymize-wasm, not from internal modules.");
+	return _TextSearch;
+};
+//#endregion
 //#region src/types.ts
 /**
 * Source of a detected entity span.
@@ -565,7 +579,37 @@ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstN
 const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
 const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
 /**
-* Load name corpus data from JSON config files.
+* Languages with per-language first/surname
+* dictionaries in @stll/anonymize-data.
+*/
+const NAME_LANGUAGES = [
+	"cs",
+	"sk",
+	"de",
+	"pl",
+	"hu",
+	"ro",
+	"fr",
+	"es",
+	"it",
+	"en",
+	"sv"
+];
+/**
+* Try importing a JSON module; return empty array
+* if not found.
+*/
+const tryImportArray = async (path) => {
+	try {
+		return (await import(path)).default;
+	} catch {
+		return [];
+	}
+};
+/**
+* Load name corpus data from per-language dictionary
+* files and legacy config files. Merges all sources.
+*
 * Safe to call multiple times; only loads once per
 * context. Must be called before detectNameCorpus or
 * the getNameCorpus*() accessors are used.
@@ -574,23 +618,40 @@ const initNameCorpus = (ctx = defaultContext) => {
 	if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
 	const promise = (async () => {
 		try {
-			const [firstMod, surnameMod, titleMod, exclusionMod] = await Promise.all([
+			const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
 				import("@stll/anonymize-data/config/names-first.json"),
 				import("@stll/anonymize-data/config/names-surnames.json"),
 				import("@stll/anonymize-data/config/names-title-tokens.json"),
 				import("@stll/anonymize-data/config/names-exclusions.json")
 			]);
-			const firstNames = firstMod.default.names;
-			const surnames = surnameMod.default.names;
+			const firstImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/first/${lang}.json`));
+			const surnameImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/surnames/${lang}.json`));
+			const [firstResults, surnameResults] = await Promise.all([Promise.all(firstImports), Promise.all(surnameImports)]);
+			const firstNames = [...legacyFirstMod.default.names];
+			for (const names of firstResults) for (const name of names) firstNames.push(name);
+			const surnames = [...legacySurnameMod.default.names];
+			for (const names of surnameResults) for (const name of names) surnames.push(name);
+			const dedup = (arr) => {
+				const seen = /* @__PURE__ */ new Set();
+				const result = [];
+				for (const item of arr) {
+					if (seen.has(item)) continue;
+					seen.add(item);
+					result.push(item);
+				}
+				return result;
+			};
+			const dedupFirst = dedup(firstNames);
+			const dedupSurnames = dedup(surnames);
 			const titles = titleMod.default.tokens;
 			const exclusions = exclusionMod.default.words;
 			ctx.nameCorpus = {
-				firstNames: Object.freeze(new Set(firstNames)),
-				surnames: Object.freeze(new Set(surnames)),
+				firstNames: Object.freeze(new Set(dedupFirst)),
+				surnames: Object.freeze(new Set(dedupSurnames)),
 				titleTokens: Object.freeze(new Set(titles)),
 				excludedWords: Object.freeze(new Set(exclusions)),
-				firstNamesList: Object.freeze(firstNames),
-				surnamesList: Object.freeze(surnames),
+				firstNamesList: Object.freeze(dedupFirst),
+				surnamesList: Object.freeze(dedupSurnames),
 				titlesList: Object.freeze(titles),
 				excludedList: Object.freeze(exclusions)
 			};
@@ -994,513 +1055,16 @@ const POST_NOMINALS = [
 	"CIPP"
 ];
 //#endregion
-//#region src/util/char-groups-data.json
-var char_groups_data_default = {
-	_comment: "Unicode character equivalence groups. Used to build regex character classes that match all variants of a character type.",
-	groups: {
-		"dash": {
-			"description": "Dash-like characters (hyphens, en-dash, em-dash, minus)",
-			"chars": [
-				{
-					"char": "-",
-					"name": "hyphen-minus",
-					"code": "U+002D"
-				},
-				{
-					"char": "–",
-					"name": "en-dash",
-					"code": "U+2013"
-				},
-				{
-					"char": "—",
-					"name": "em-dash",
-					"code": "U+2014"
-				},
-				{
-					"char": "‐",
-					"name": "hyphen",
-					"code": "U+2010"
-				},
-				{
-					"char": "‑",
-					"name": "non-breaking hyphen",
-					"code": "U+2011"
-				},
-				{
-					"char": "−",
-					"name": "minus sign",
-					"code": "U+2212"
-				},
-				{
-					"char": "⁃",
-					"name": "hyphen bullet",
-					"code": "U+2043"
-				},
-				{
-					"char": "‒",
-					"name": "figure dash",
-					"code": "U+2012"
-				},
-				{
-					"char": "―",
-					"name": "horizontal bar",
-					"code": "U+2015"
-				},
-				{
-					"char": "⸺",
-					"name": "two-em dash",
-					"code": "U+2E3A"
-				},
-				{
-					"char": "⸻",
-					"name": "three-em dash",
-					"code": "U+2E3B"
-				},
-				{
-					"char": "־",
-					"name": "Hebrew maqaf",
-					"code": "U+05BE"
-				}
-			]
-		},
-		"space": {
-			"description": "Space-like characters",
-			"chars": [
-				{
-					"char": " ",
-					"name": "space",
-					"code": "U+0020"
-				},
-				{
-					"char": "\xA0",
-					"name": "no-break space",
-					"code": "U+00A0"
-				},
-				{
-					"char": " ",
-					"name": "en space",
-					"code": "U+2002"
-				},
-				{
-					"char": " ",
-					"name": "em space",
-					"code": "U+2003"
-				},
-				{
-					"char": " ",
-					"name": "thin space",
-					"code": "U+2009"
-				},
-				{
-					"char": " ",
-					"name": "hair space",
-					"code": "U+200A"
-				},
-				{
-					"char": " ",
-					"name": "narrow no-break space",
-					"code": "U+202F"
-				},
-				{
-					"char": " ",
-					"name": "Ogham space mark",
-					"code": "U+1680"
-				},
-				{
-					"char": " ",
-					"name": "en quad",
-					"code": "U+2000"
-				},
-				{
-					"char": " ",
-					"name": "em quad",
-					"code": "U+2001"
-				},
-				{
-					"char": " ",
-					"name": "three-per-em space",
-					"code": "U+2004"
-				},
-				{
-					"char": " ",
-					"name": "four-per-em space",
-					"code": "U+2005"
-				},
-				{
-					"char": " ",
-					"name": "six-per-em space",
-					"code": "U+2006"
-				},
-				{
-					"char": " ",
-					"name": "figure space",
-					"code": "U+2007"
-				},
-				{
-					"char": " ",
-					"name": "punctuation space",
-					"code": "U+2008"
-				},
-				{
-					"char": " ",
-					"name": "medium mathematical space",
-					"code": "U+205F"
-				},
-				{
-					"char": "　",
-					"name": "ideographic space",
-					"code": "U+3000"
-				},
-				{
-					"char": "​",
-					"name": "zero width space",
-					"code": "U+200B"
-				}
-			]
-		},
-		"quote-double": {
-			"description": "Double quotation marks",
-			"chars": [
-				{
-					"char": "\"",
-					"name": "quotation mark",
-					"code": "U+0022"
-				},
-				{
-					"char": "“",
-					"name": "left double quotation mark",
-					"code": "U+201C"
-				},
-				{
-					"char": "”",
-					"name": "right double quotation mark",
-					"code": "U+201D"
-				},
-				{
-					"char": "„",
-					"name": "double low-9 quotation mark",
-					"code": "U+201E"
-				},
-				{
-					"char": "«",
-					"name": "left guillemet",
-					"code": "U+00AB"
-				},
-				{
-					"char": "»",
-					"name": "right guillemet",
-					"code": "U+00BB"
-				}
-			]
-		},
-		"quote-single": {
-			"description": "Single quotation marks and apostrophes",
-			"chars": [
-				{
-					"char": "'",
-					"name": "apostrophe",
-					"code": "U+0027"
-				},
-				{
-					"char": "‘",
-					"name": "left single quotation mark",
-					"code": "U+2018"
-				},
-				{
-					"char": "’",
-					"name": "right single quotation mark",
-					"code": "U+2019"
-				},
-				{
-					"char": "‚",
-					"name": "single low-9 quotation mark",
-					"code": "U+201A"
-				},
-				{
-					"char": "‹",
-					"name": "left single guillemet",
-					"code": "U+2039"
-				},
-				{
-					"char": "›",
-					"name": "right single guillemet",
-					"code": "U+203A"
-				},
-				{
-					"char": "ʼ",
-					"name": "modifier letter apostrophe",
-					"code": "U+02BC"
-				},
-				{
-					"char": "´",
-					"name": "acute accent",
-					"code": "U+00B4"
-				},
-				{
-					"char": "`",
-					"name": "grave accent",
-					"code": "U+0060"
-				},
-				{
-					"char": "ʽ",
-					"name": "modifier letter reversed comma",
-					"code": "U+02BD"
-				},
-				{
-					"char": "ʾ",
-					"name": "modifier letter right half ring",
-					"code": "U+02BE"
-				},
-				{
-					"char": "ʿ",
-					"name": "modifier letter left half ring",
-					"code": "U+02BF"
-				},
-				{
-					"char": "ˈ",
-					"name": "modifier letter vertical line",
-					"code": "U+02C8"
-				}
-			]
-		},
-		"dot": {
-			"description": "Dot-like characters",
-			"chars": [
-				{
-					"char": ".",
-					"name": "full stop",
-					"code": "U+002E"
-				},
-				{
-					"char": "·",
-					"name": "middle dot",
-					"code": "U+00B7"
-				},
-				{
-					"char": "•",
-					"name": "bullet",
-					"code": "U+2022"
-				},
-				{
-					"char": "․",
-					"name": "one dot leader",
-					"code": "U+2024"
-				},
-				{
-					"char": "。",
-					"name": "CJK full stop",
-					"code": "U+3002"
-				},
-				{
-					"char": "׃",
-					"name": "Hebrew sof pasuq",
-					"code": "U+05C3"
-				}
-			]
-		},
-		"slash": {
-			"description": "Slash-like characters",
-			"chars": [
-				{
-					"char": "/",
-					"name": "solidus",
-					"code": "U+002F"
-				},
-				{
-					"char": "⁄",
-					"name": "fraction slash",
-					"code": "U+2044"
-				},
-				{
-					"char": "∕",
-					"name": "division slash",
-					"code": "U+2215"
-				}
-			]
-		},
-		"colon": {
-			"description": "Colon-like characters",
-			"chars": [
-				{
-					"char": ":",
-					"name": "colon",
-					"code": "U+003A"
-				},
-				{
-					"char": "꞉",
-					"name": "modifier letter colon",
-					"code": "U+A789"
-				},
-				{
-					"char": "∶",
-					"name": "ratio",
-					"code": "U+2236"
-				},
-				{
-					"char": "：",
-					"name": "full-width colon",
-					"code": "U+FF1A"
-				},
-				{
-					"char": "፥",
-					"name": "Ethiopic colon",
-					"code": "U+1365"
-				}
-			]
-		},
-		"comma": {
-			"description": "Comma-like characters",
-			"chars": [
-				{
-					"char": ",",
-					"name": "comma",
-					"code": "U+002C"
-				},
-				{
-					"char": "‚",
-					"name": "single low-9 quotation mark (used as comma)",
-					"code": "U+201A"
-				},
-				{
-					"char": "،",
-					"name": "Arabic comma",
-					"code": "U+060C"
-				},
-				{
-					"char": "、",
-					"name": "ideographic comma",
-					"code": "U+3001"
-				},
-				{
-					"char": "，",
-					"name": "full-width comma",
-					"code": "U+FF0C"
-				},
-				{
-					"char": "﹐",
-					"name": "small comma",
-					"code": "U+FE50"
-				},
-				{
-					"char": "՝",
-					"name": "Armenian comma",
-					"code": "U+055D"
-				}
-			]
-		},
-		"ellipsis": {
-			"description": "Ellipsis characters",
-			"chars": [{
-				"char": "…",
-				"name": "horizontal ellipsis",
-				"code": "U+2026"
-			}]
-		},
-		"question-mark": {
-			"description": "Question mark variants",
-			"chars": [
-				{
-					"char": "?",
-					"name": "question mark",
-					"code": "U+003F"
-				},
-				{
-					"char": "？",
-					"name": "full-width question mark",
-					"code": "U+FF1F"
-				},
-				{
-					"char": "⁇",
-					"name": "double question mark",
-					"code": "U+2047"
-				},
-				{
-					"char": "⁈",
-					"name": "question exclamation mark",
-					"code": "U+2048"
-				},
-				{
-					"char": "؟",
-					"name": "Arabic question mark",
-					"code": "U+061F"
-				},
-				{
-					"char": "‽",
-					"name": "interrobang",
-					"code": "U+203D"
-				}
-			]
-		},
-		"exclamation-mark": {
-			"description": "Exclamation mark variants",
-			"chars": [
-				{
-					"char": "!",
-					"name": "exclamation mark",
-					"code": "U+0021"
-				},
-				{
-					"char": "！",
-					"name": "full-width exclamation mark",
-					"code": "U+FF01"
-				},
-				{
-					"char": "‼",
-					"name": "double exclamation mark",
-					"code": "U+203C"
-				},
-				{
-					"char": "¡",
-					"name": "inverted exclamation mark",
-					"code": "U+00A1"
-				}
-			]
-		},
-		"semicolon": {
-			"description": "Semicolon variants",
-			"chars": [
-				{
-					"char": ";",
-					"name": "semicolon",
-					"code": "U+003B"
-				},
-				{
-					"char": "؛",
-					"name": "Arabic semicolon",
-					"code": "U+061B"
-				},
-				{
-					"char": "；",
-					"name": "full-width semicolon",
-					"code": "U+FF1B"
-				},
-				{
-					"char": "፤",
-					"name": "Ethiopic semicolon",
-					"code": "U+1364"
-				}
-			]
-		},
-		"underscore": {
-			"description": "Underscore variants",
-			"chars": [{
-				"char": "_",
-				"name": "low line",
-				"code": "U+005F"
-			}, {
-				"char": "＿",
-				"name": "full-width low line",
-				"code": "U+FF3F"
-			}]
-		}
-	}
-};
-//#endregion
 //#region src/util/char-groups.ts
 /** Chars that need escaping inside a regex char class. */
 const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
 const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
-const loadConfig = () => char_groups_data_default;
+let cached;
+const loadConfig = () => {
+	if (cached) return cached;
+	cached = __require("@stll/anonymize-data/config/char-groups.json");
+	return cached;
+};
 /**
 * Get the raw characters for a named group.
 * Throws if the group does not exist.
@@ -1696,7 +1260,7 @@ const CZ_BIRTH_NUMBER = {
 	validator: cz.rc
 };
 const DATE_NUMERIC = {
-	pattern: "\\b(?:\\d{1,2}[./]\\d{1,2}[./]\\d{2,4}|\\d{4}-\\d{2}-\\d{2})\\b",
+	pattern: "\\b(?:\\d{1,2}[./]\\d{1,2}[./]\\d{2,4}|\\d{4}-\\d{2}-\\d{2}|\\d{4}\\.\\d{2}\\.\\d{2})\\b",
 	label: "date",
 	score: 1
 };
@@ -1782,6 +1346,11 @@ const ALL_REGEX_DEFS = [
 		label: "url",
 		score: .9
 	},
+	{
+		pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
+		label: "date",
+		score: .9
+	},
 	...STDNUM_ENTRIES
 ];
 /** Flat pattern array for text-search. */
@@ -2765,6 +2334,7 @@ const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
 const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
 const HAS_DIGIT_RE = /\d/;
 const ADDRESS_COMPONENTS_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|č\.p\.|č\.ev\.|č\.|sídliště|bulvár)(?=[\s,./]|$)/i;
+const JURISDICTION_RE = /^(?:state|commonwealth|district|territory)\s+of\s+/i;
 const MAX_ENTITY_LENGTH = {
 	organization: 80,
 	person: 60
@@ -2816,8 +2386,8 @@ const filterFalsePositives = (entities, ctx = defaultContext) => {
 		if (STANDALONE_YEAR_RE.test(trimmed)) continue;
 		if (entity.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
 		if ((entity.label === "person" || entity.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
-		if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
-		if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
+		if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
+		if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
 		filtered.push(entity);
 	}
 	return filtered;
@@ -4090,7 +3660,7 @@ const loadRules = async () => {
 			mapping.push(ruleIdx);
 		}
 	}
-	const builtSearch = patterns.length > 0 ? new TextSearch(patterns, {
+	const builtSearch = patterns.length > 0 ? new (getTextSearch())(patterns, {
 		overlapStrategy: "all",
 		caseInsensitive: true,
 		wholeWords: true
@@ -4482,24 +4052,6 @@ const enforceBoundaryConsistency = (entities, fullText) => {
 };
 //#endregion
 //#region src/build-unified-search.ts
-/**
-* Build the unified search instances from all
-* detector pattern sources.
-*
-* Two TextSearch instances (not one) to avoid
-* 200K per-pattern object allocations:
-* 1. regex + triggers + legal-forms (mixed, ~140
-*    patterns, caseInsensitive for trigger AC)
-* 2. deny-list + street-types + gazetteer
-*    (caseInsensitive, overlap "all";
-*    deny-list/street-type use per-pattern
-*    wholeWords: true; gazetteer exact use
-*    wholeWords: false; gazetteer fuzzy use
-*    distance: 2 via @stll/fuzzy-search)
-*
-* All patterns are PatternEntry objects with
-* per-pattern literal/wholeWords settings.
-*/
 const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
 	const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
 		buildLegalFormPatterns(),
@@ -4545,11 +4097,12 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
 		literal: true,
 		caseInsensitive: true
 	}));
-	const tsRegex = new TextSearch([
+	const regexAllPatterns = [
 		...allRegex,
 		...legalForms,
 		...triggerEntries
-	]);
+	];
+	const tsRegex = new (getTextSearch())(regexAllPatterns);
 	offset = 0;
 	const denyListOriginals = denyListData?.originals ?? [];
 	const denyListSlice = {
@@ -4580,10 +4133,10 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
 	];
 	return {
 		tsRegex,
-		tsLiterals: literalAllPatterns.length > 0 ? new TextSearch(literalAllPatterns, {
+		tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
 			caseInsensitive: true,
 			overlapStrategy: "all"
-		}) : new TextSearch([]),
+		}) : new (getTextSearch())([]),
 		slices: {
 			regex: regexSlice,
 			legalForms: legalFormsSlice,
@@ -5551,6 +5104,9 @@ const levenshtein = (rawA, rawB) => {
 	return row[aLen] ?? 0;
 };
 //#endregion
+//#region src/index.ts
+initTextSearch(TextSearch);
+//#endregion
 export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
 
-//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.mjs.map