@stll/anonymize-wasm 1.0.9 → 1.1.0

package/dist/wasm.d.mts

@@ -174,6 +174,59 @@ type RedactionResult = {
  * Configuration for the detection pipeline.
  */
 type DenyListCategory = "Names" | "Places" | "Addresses" | "Courts" | "Financial" | "Government" | "Healthcare" | "Education" | "Political" | "Organizations" | "International";
+/**
+ * Metadata for a single dictionary entry in the
+ * deny-list system. Mirrors the shape from
+ * `@stll/anonymize-data` so consumers can pass
+ * pre-loaded data without the runtime dependency.
+ */
+type DictionaryMeta = {
+  label: string;
+  category: DenyListCategory;
+  country: string | null;
+};
+/**
+ * Pre-loaded dictionary data for dependency injection.
+ * Consumers that want name/city/deny-list detection
+ * import from `@stll/anonymize-data` themselves and
+ * pass the data here; the anonymize package itself
+ * has zero cross-package imports.
+ *
+ * All fields are optional. When a field is absent,
+ * the corresponding detection path is skipped (same
+ * behavior as when `@stll/anonymize-data` was not
+ * installed).
+ */
+type Dictionaries = {
+  /**
+   * First names per language code (e.g., "cs", "de").
+   * Merged with legacy config names at init time.
+   */
+  firstNames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Surnames per language code.
+   * Merged with legacy config names at init time.
+   */
+  surnames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Pre-loaded deny-list dictionaries keyed by
+   * dictionary ID (e.g., "courts/CZ", "banks/DE").
+   * Each value is the array of terms for that
+   * dictionary.
+   */
+  denyList?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Metadata per dictionary ID. Required when
+   * `denyList` is provided so the pipeline knows
+   * labels, categories, and country filters.
+   */
+  denyListMeta?: Readonly<Record<string, DictionaryMeta>>;
+  /**
+   * Pre-loaded city names, already merged across
+   * all desired countries.
+   */
+  cities?: readonly string[];
+};
 type PipelineConfig = {
   threshold: number;
   enableTriggerPhrases: boolean;
@@ -209,6 +262,13 @@ type PipelineConfig = {
    */
   labels: string[];
   workspaceId: string;
+  /**
+   * Pre-loaded dictionary data for name, deny-list,
+   * and city detection. When omitted, dictionary-based
+   * detection paths are skipped. Consumers load from
+   * `@stll/anonymize-data` and pass the data here.
+   */
+  dictionaries?: Dictionaries;
 };
 /**
  * Canonical entity labels used across the pipeline.
@@ -261,7 +321,7 @@ declare const CURRENCY_PATTERN_META: Readonly<RegexMeta>;
 declare const processRegexMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, meta_: readonly RegexMeta[]) => Entity[];
 //#endregion
 //#region src/detectors/deny-list.d.ts
-type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories">;
+type DenyListConfig = Pick<PipelineConfig, "enableDenyList" | "enableNameCorpus" | "denyListCountries" | "denyListRegions" | "denyListExcludeCategories" | "dictionaries">;
 /**
  * Source tag for each pattern in the automaton.
  * "deny-list" = standard deny list entry
@@ -289,13 +349,13 @@ type DenyListData = {
 };
 /**
  * Resolve which dictionaries to load based on country
- * and category filters, load them, and build the deny
- * list data. The returned data provides PatternEntry[]
- * for the unified builder and parallel arrays for
+ * and category filters, then build the deny list data.
+ * The returned data provides PatternEntry[] for the
+ * unified builder and parallel arrays for
  * post-processing.
  *
- * Requires `@stll/anonymize-data` to be installed.
- * Returns null if the data package is not available.
+ * Dictionary data is injected via `config.dictionaries`.
+ * Returns null if no dictionaries are provided.
  */
 declare const buildDenyList: (config: DenyListConfig, ctx?: PipelineContext) => Promise<DenyListData | null>;
 /**
@@ -306,7 +366,7 @@ declare const buildDenyList: (config: DenyListConfig, ctx?: PipelineContext) =>
  * the search instance was built on a different context
  * (e.g. cachedSearch).
  */
-declare const ensureDenyListData: (ctx?: PipelineContext) => Promise<void>;
+declare const ensureDenyListData: (ctx?: PipelineContext, dictionaries?: Dictionaries) => Promise<void>;
 /**
  * Process deny list matches from the unified search.
  * Receives all matches; filters to the deny list slice
@@ -647,14 +707,18 @@ declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entit
 //#endregion
 //#region src/detectors/names.d.ts
 /**
- * Load name corpus data from per-language dictionary
- * files and legacy config files. Merges all sources.
+ * Load name corpus data from injected dictionaries
+ * and legacy config files. Merges all sources.
  *
  * Safe to call multiple times; only loads once per
  * context. Must be called before detectNameCorpus or
  * the getNameCorpus*() accessors are used.
+ *
+ * @param dictionaries Optional pre-loaded dictionaries
+ *   with per-language first names and surnames. When
+ *   omitted, only legacy config files are used.
  */
-declare const initNameCorpus: (ctx?: PipelineContext) => Promise<void>;
+declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionaries) => Promise<void>;
 /**
  * Detect person names by looking up tokens against the
  * name corpus, then chaining adjacent name-like tokens.
@@ -933,5 +997,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
  */
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
-export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
+export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
 //# sourceMappingURL=wasm.d.mts.map

package/dist/wasm.mjs

@@ -606,42 +606,18 @@ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstN
 const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
 const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
 /**
-* Languages with per-language first/surname
-* dictionaries from @stll/anonymize-data (optional).
-*/
-const NAME_LANGUAGES = [
-	"cs",
-	"sk",
-	"de",
-	"pl",
-	"hu",
-	"ro",
-	"fr",
-	"es",
-	"it",
-	"en",
-	"sv"
-];
-/**
-* Try importing a JSON module; return empty array
-* if not found.
-*/
-const tryImportArray = async (path) => {
-	try {
-		return (await import(path)).default;
-	} catch {
-		return [];
-	}
-};
-/**
-* Load name corpus data from per-language dictionary
-* files and legacy config files. Merges all sources.
+* Load name corpus data from injected dictionaries
+* and legacy config files. Merges all sources.
 *
 * Safe to call multiple times; only loads once per
 * context. Must be called before detectNameCorpus or
 * the getNameCorpus*() accessors are used.
+*
+* @param dictionaries Optional pre-loaded dictionaries
+*   with per-language first names and surnames. When
+*   omitted, only legacy config files are used.
 */
-const initNameCorpus = (ctx = defaultContext) => {
+const initNameCorpus = (ctx = defaultContext, dictionaries) => {
 	if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
 	const promise = (async () => {
 		try {
@@ -651,13 +627,10 @@ const initNameCorpus = (ctx = defaultContext) => {
 				import("./names-title-tokens.mjs"),
 				import("./names-exclusions.mjs")
 			]);
-			const firstImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/first/${lang}.json`));
-			const surnameImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/surnames/${lang}.json`));
-			const [firstResults, surnameResults] = await Promise.all([Promise.all(firstImports), Promise.all(surnameImports)]);
 			const firstNames = [...legacyFirstMod.default.names];
-			for (const names of firstResults) for (const name of names) firstNames.push(name);
+			if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
 			const surnames = [...legacySurnameMod.default.names];
-			for (const names of surnameResults) for (const name of names) surnames.push(name);
+			if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
 			const dedup = (arr) => {
 				const seen = /* @__PURE__ */ new Set();
 				const result = [];
@@ -3156,17 +3129,6 @@ const normalizeForSearch = (text) => {
 };
 //#endregion
 //#region src/detectors/deny-list.ts
-/**
-* Try to load the optional @stll/anonymize-data package.
-* Returns null if not installed.
-*/
-const loadDataModule = async () => {
-	try {
-		return await import("./dist.mjs");
-	} catch {
-		return null;
-	}
-};
 const loadAllowList = (ctx) => {
 	if (ctx.allowListPromise) return ctx.allowListPromise;
 	ctx.allowListPromise = (async () => {
@@ -3285,46 +3247,33 @@ const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
 const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
 /**
 * Resolve which dictionaries to load based on country
-* and category filters, load them, and build the deny
-* list data. The returned data provides PatternEntry[]
-* for the unified builder and parallel arrays for
+* and category filters, then build the deny list data.
+* The returned data provides PatternEntry[] for the
+* unified builder and parallel arrays for
 * post-processing.
 *
-* Requires `@stll/anonymize-data` to be installed.
-* Returns null if the data package is not available.
+* Dictionary data is injected via `config.dictionaries`.
+* Returns null if no dictionaries are provided.
 */
 const buildDenyList = async (config, ctx = defaultContext) => {
-	await initNameCorpus(ctx);
+	await initNameCorpus(ctx, config.dictionaries);
 	await Promise.all([
 		loadStopwords(ctx),
 		loadAllowList(ctx),
 		loadPersonStopwords(ctx),
 		loadGenericRoles(ctx)
 	]);
-	const dataModule = await loadDataModule();
-	if (!dataModule) return null;
+	const dictionaries = config.dictionaries;
+	const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
+	const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
+	if (!hasDenyList && !hasCities) return buildNameCorpusOnly(config, ctx);
 	const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
 	const excluded = config.denyListExcludeCategories;
 	const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
-	const ids = [...dataModule.ALL_DICTIONARY_IDS].filter((id) => {
-		const meta = dataModule.DICTIONARY_META[id];
-		if (!meta) return false;
-		if (!config.enableNameCorpus && meta.category === "Names") return false;
-		if (excludeCategories.has(meta.category)) return false;
-		if (allowedCountries === null) return true;
-		if (meta.country === null) return true;
-		return allowedCountries.has(meta.country);
-	});
 	const patternList = [];
 	const labelList = [];
 	const sourceList = [];
 	const patternIndex = /* @__PURE__ */ new Map();
-	const results = await Promise.all(ids.map(async (id) => {
-		return {
-			id,
-			entries: await dataModule.loadDictionary(id)
-		};
-	}));
 	const addDenyListEntry = (entry, label) => {
 		const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
 		if (normalized.length === 0) return;
@@ -3340,47 +3289,59 @@ const buildDenyList = async (config, ctx = defaultContext) => {
 			sourceList.push(["deny-list"]);
 		}
 	};
-	for (const { id, entries } of results) {
-		const meta = dataModule.DICTIONARY_META[id];
-		if (!meta) continue;
-		for (const entry of entries) addDenyListEntry(entry, meta.label);
-	}
-	if (!excludeCategories.has("Places")) {
-		const cityCountries = allowedCountries !== null ? [...allowedCountries] : [
-			"AT",
-			"AU",
-			"BE",
-			"BG",
-			"BR",
-			"CA",
-			"CH",
-			"CZ",
-			"DE",
-			"DK",
-			"ES",
-			"FI",
-			"FR",
-			"GB",
-			"GR",
-			"HR",
-			"HU",
-			"IE",
-			"IT",
-			"LU",
-			"NL",
-			"NO",
-			"NZ",
-			"PL",
-			"PT",
-			"RO",
-			"SE",
-			"SI",
-			"SK",
-			"US"
-		];
-		const cityEntries = await dataModule.loadCityDictionaries(cityCountries);
-		for (const entry of cityEntries) addDenyListEntry(entry, "address");
+	if (hasDenyList) {
+		const denyListData = dictionaries.denyList;
+		const metaData = dictionaries.denyListMeta;
+		for (const [id, entries] of Object.entries(denyListData)) {
+			const meta = metaData[id];
+			if (!meta) continue;
+			if (!config.enableNameCorpus && meta.category === "Names") continue;
+			if (excludeCategories.has(meta.category)) continue;
+			if (allowedCountries !== null && meta.country !== null) {
+				if (!allowedCountries.has(meta.country)) continue;
+			}
+			for (const entry of entries) addDenyListEntry(entry, meta.label);
+		}
 	}
+	if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
+	appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, patternIndex);
+	if (patternList.length === 0) return null;
+	return {
+		labels: labelList,
+		originals: patternList,
+		sources: sourceList
+	};
+};
+/**
+* Build deny-list data containing only name corpus
+* entries (no external dictionaries). Used when no
+* dictionary data is injected but name corpus is
+* enabled.
+*/
+const buildNameCorpusOnly = (config, ctx) => {
+	if (!config.enableNameCorpus) return null;
+	const excluded = config.denyListExcludeCategories;
+	if ((excluded ? new Set(excluded) : /* @__PURE__ */ new Set()).has("Names")) return null;
+	const patternList = [];
+	const labelList = [];
+	const sourceList = [];
+	appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, /* @__PURE__ */ new Map());
+	if (patternList.length === 0) return null;
+	return {
+		labels: labelList,
+		originals: patternList,
+		sources: sourceList
+	};
+};
+/**
+* Append name corpus entries (first names, surnames,
+* titles) to the pattern arrays. Shared between
+* buildDenyList and buildNameCorpusOnly.
+*/
+const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList, patternIndex) => {
+	const excluded = config.denyListExcludeCategories;
+	const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
+	if (!config.enableNameCorpus || excludeCategories.has("Names")) return;
 	const addNameEntry = (name, source) => {
 		const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
 		if (normalized.length === 0) return;
@@ -3396,30 +3357,22 @@ const buildDenyList = async (config, ctx = defaultContext) => {
 			sourceList.push([source]);
 		}
 	};
-	if (config.enableNameCorpus && !excludeCategories.has("Names")) {
-		for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
-		for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
-		for (const title of getNameCorpusTitles(ctx)) {
-			const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
-			if (norm.length === 0) continue;
-			const lower = norm.toLowerCase();
-			const existing = patternIndex.get(lower);
-			if (existing !== void 0) {
-				if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
-			} else {
-				patternIndex.set(lower, patternList.length);
-				patternList.push(norm);
-				labelList.push(["person"]);
-				sourceList.push(["title"]);
-			}
+	for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
+	for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
+	for (const title of getNameCorpusTitles(ctx)) {
+		const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
+		if (norm.length === 0) continue;
+		const lower = norm.toLowerCase();
+		const existing = patternIndex.get(lower);
+		if (existing !== void 0) {
+			if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
+		} else {
+			patternIndex.set(lower, patternList.length);
+			patternList.push(norm);
+			labelList.push(["person"]);
+			sourceList.push(["title"]);
 		}
 	}
-	if (patternList.length === 0) return null;
-	return {
-		labels: labelList,
-		originals: patternList,
-		sources: sourceList
-	};
 };
 /**
 * Ensure all deny-list support data (stopwords, allow
@@ -3429,8 +3382,8 @@ const buildDenyList = async (config, ctx = defaultContext) => {
 * the search instance was built on a different context
 * (e.g. cachedSearch).
 */
-const ensureDenyListData = async (ctx = defaultContext) => {
-	await initNameCorpus(ctx);
+const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
+	await initNameCorpus(ctx, dictionaries);
 	await Promise.all([
 		loadStopwords(ctx),
 		loadAllowList(ctx),
@@ -5186,7 +5139,7 @@ const runPipeline = async (options) => {
 		initStreetAbbrevs(),
 		hotwordInit
 	]);
-	if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx);
+	if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
 	let zones = [];
 	if (config.enableZoneClassification && zoneInitOk) {
 		zones = classifyZones(fullText, ctx);
@@ -5212,7 +5165,7 @@ const runPipeline = async (options) => {
 	let rawNameCorpusEntities = [];
 	let nameCorpusEntities = [];
 	if (config.enableNameCorpus && !config.enableDenyList) {
-		await initNameCorpus(ctx);
+		await initNameCorpus(ctx, config.dictionaries);
 		checkAbort(signal);
 		rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
 		nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);