@stll/anonymize 2.8.3 → 2.9.1

package/dist/native-node2.mjs

@@ -1,4 +1,5 @@
 import { PreparedNativeAnonymizer, PreparedNativePipeline, assertNativeBindingVersion, convert_external_detection_batch as convert_external_detection_batch$1, createNativePipelineFromPackage, diagnostics_json as diagnostics_json$1, diagnostics_stream_json as diagnostics_stream_json$1, isNativeAnonymizeBinding, load_prepared_package as load_prepared_package$1, native_package_version as native_package_version$1, normalize_for_search as normalize_for_search$1, prepare_search_package as prepare_search_package$1, redact_text as redact_text$1, redact_text_json as redact_text_json$1, redact_text_stream_json as redact_text_stream_json$1, summary_diagnostics_json as summary_diagnostics_json$1 } from "./native.mjs";
+import { n as applyPipelineLanguageScope, r as language_scopes_default, t as defaultDictionaryBundleOptions } from "./build-native-package2.mjs";
 import { DEFAULT_ENTITY_LABELS } from "./constants.mjs";
 import { createRequire } from "node:module";
 import { existsSync, readFileSync, readdirSync } from "node:fs";
@@ -44,146 +45,6 @@ const createPipelineContext = () => ({
 */
 const defaultContext = createPipelineContext();
 //#endregion
-//#region src/language-scope.ts
-const scopeData = {
-	_comment: "Default dictionary scopes for content language hints. Lower-level caller config can still override name corpus languages and deny-list countries independently.",
-	languages: {
-		"cs": {
-			"nameCorpusLanguages": ["cs", "sk"],
-			"denyListCountries": ["CZ", "SK"]
-		},
-		"de": {
-			"nameCorpusLanguages": ["de"],
-			"denyListCountries": [
-				"DE",
-				"AT",
-				"CH"
-			]
-		},
-		"en": {
-			"nameCorpusLanguages": ["en"],
-			"denyListCountries": [
-				"US",
-				"GB",
-				"CA",
-				"AU",
-				"IE"
-			]
-		},
-		"es": {
-			"nameCorpusLanguages": ["es"],
-			"denyListCountries": [
-				"ES",
-				"MX",
-				"AR",
-				"CL",
-				"CO",
-				"PE",
-				"EC",
-				"VE",
-				"UY",
-				"PY",
-				"BO",
-				"CR",
-				"PA",
-				"DO",
-				"GT",
-				"HN",
-				"SV",
-				"NI",
-				"CU"
-			]
-		},
-		"fr": {
-			"nameCorpusLanguages": ["fr"],
-			"denyListCountries": [
-				"FR",
-				"BE",
-				"CH",
-				"CA",
-				"LU",
-				"MC"
-			]
-		},
-		"hu": {
-			"nameCorpusLanguages": ["hu"],
-			"denyListCountries": ["HU"]
-		},
-		"it": {
-			"nameCorpusLanguages": ["it"],
-			"denyListCountries": ["IT", "CH"]
-		},
-		"lv": {
-			"nameCorpusLanguages": [],
-			"denyListCountries": ["LV"]
-		},
-		"pl": {
-			"nameCorpusLanguages": ["pl"],
-			"denyListCountries": ["PL"]
-		},
-		"pt-br": {
-			"nameCorpusLanguages": ["pt-br"],
-			"denyListCountries": ["BR"]
-		},
-		"ro": {
-			"nameCorpusLanguages": ["ro"],
-			"denyListCountries": ["RO", "MD"]
-		},
-		"sk": {
-			"nameCorpusLanguages": ["sk", "cs"],
-			"denyListCountries": ["SK", "CZ"]
-		},
-		"sv": {
-			"nameCorpusLanguages": ["sv"],
-			"denyListCountries": ["SE", "FI"]
-		}
-	}
-};
-const normalizeLanguage = (language) => language.trim().toLowerCase();
-const fallbackLanguage = (language) => {
-	const index = language.indexOf("-");
-	return index === -1 ? null : language.slice(0, index);
-};
-const uniquePush = (target, values) => {
-	const seen = new Set(target);
-	for (const value of values) {
-		if (seen.has(value)) continue;
-		seen.add(value);
-		target.push(value);
-	}
-};
-const resolveLanguageScope = (language) => {
-	const normalized = normalizeLanguage(language);
-	if (normalized.length === 0) return null;
-	const exact = scopeData.languages[normalized];
-	if (exact !== void 0) return exact;
-	const fallback = fallbackLanguage(normalized);
-	return fallback === null ? null : scopeData.languages[fallback] ?? null;
-};
-const configuredLanguages = (config) => {
-	if (config.languages !== void 0) return config.languages;
-	return config.language === void 0 ? [] : [config.language];
-};
-const applyPipelineLanguageScope = (config) => {
-	const languages = configuredLanguages(config);
-	if (languages.length === 0) return config;
-	const nameCorpusLanguages = [];
-	const denyListCountries = [];
-	for (const language of languages) {
-		const scope = resolveLanguageScope(language);
-		if (scope === null) continue;
-		uniquePush(nameCorpusLanguages, scope.nameCorpusLanguages ?? []);
-		uniquePush(denyListCountries, scope.denyListCountries ?? []);
-	}
-	const next = {};
-	if (config.nameCorpusLanguages === void 0 && nameCorpusLanguages.length > 0) next.nameCorpusLanguages = nameCorpusLanguages;
-	if (config.denyListCountries === void 0 && denyListCountries.length > 0) next.denyListCountries = denyListCountries;
-	return Object.keys(next).length === 0 ? config : {
-		...config,
-		...next
-	};
-};
-//#endregion
 //#region src/types.ts
 const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
 //#endregion
@@ -198,7 +59,8 @@ const languageSelectionKey = (languages) => {
 //#region src/pipeline-cache-key.ts
 const DEFAULT_CUSTOM_REGEX_SCORE = .9;
 const contentLanguageFingerprint = (config) => {
-	return languageSelectionKey(config.languages ?? (config.language === void 0 ? [] : [config.language]));
+	const languages = config.languages ?? (config.language === void 0 ? [] : [config.language]);
+	return languageSelectionKey(languages);
 };
 const pipelineConfigKey = (config, gazetteerEntries) => {
 	const legalFormsEnabled = isLegalFormsEnabled(config);
@@ -308,14 +170,15 @@ const prepareNativePipelinePackage = async ({ binding, config, gazetteerEntries
 	return new Uint8Array(packageBytes);
 };
 const createNativePipelineFromConfig = async ({ binding, config, gazetteerEntries = [], context }) => {
+	const packageBytes = await getCachedNativePipelinePackage({
+		binding,
+		config,
+		gazetteerEntries,
+		...context ? { context } : {}
+	});
 	return createNativePipelineFromPackage({
 		binding,
-		packageBytes: await getCachedNativePipelinePackage({
-			binding,
-			config,
-			gazetteerEntries,
-			...context ? { context } : {}
-		})
+		packageBytes
 	});
 };
 const getCachedNativePipelinePackage = async ({ binding, config, gazetteerEntries = [], context, compressed = false }) => {
@@ -395,6 +258,98 @@ const DEFAULT_NATIVE_PIPELINE_CONFIG = {
 	workspaceId: "native-pipeline-default"
 };
 //#endregion
+//#region src/pipeline-language.ts
+const isSupportedLanguage = (language) => Object.hasOwn(language_scopes_default.languages, language);
+const SUPPORTED_LANGUAGES = Object.freeze(Object.keys(language_scopes_default.languages).filter(isSupportedLanguage).toSorted());
+const normalizeLanguage = (language) => {
+	if (typeof language !== "string") throw new TypeError("Pipeline language codes must be strings");
+	const normalized = language.trim().toLowerCase();
+	if (!isSupportedLanguage(normalized)) throw new RangeError(`Unsupported pipeline language ${JSON.stringify(language)}; expected one of: ${SUPPORTED_LANGUAGES.join(", ")}`);
+	return normalized;
+};
+const normalizePipelineLanguageSelection = (selection) => {
+	if (selection === void 0 || typeof selection === "string" && selection.trim().toLowerCase() === "all") return { type: "all" };
+	const requested = Array.isArray(selection) ? selection : [selection];
+	if (requested.length === 0) throw new RangeError("Pipeline language selection must not be empty");
+	const normalized = [...new Set(requested.map(normalizeLanguage))].toSorted();
+	const first = normalized.at(0);
+	if (first === void 0) throw new RangeError("Pipeline language selection must not be empty");
+	return {
+		type: "languages",
+		languages: [first, ...normalized.slice(1)]
+	};
+};
+const pipelineLanguageSelectionKey = (selection) => selection.type === "all" ? "all" : selection.languages.join(",");
+//#endregion
+//#region src/create-pipeline.ts
+const dictionaryCache = /* @__PURE__ */ new Map();
+const semanticPipelineCache = /* @__PURE__ */ new WeakMap();
+const MAX_SEMANTIC_PIPELINE_CACHE_ENTRIES = 8;
+const getCachedEntry = (cache, key) => {
+	const cached = cache.get(key);
+	if (cached === void 0) return;
+	cache.delete(key);
+	cache.set(key, cached);
+	return cached;
+};
+const setCachedEntry = (cache, key, value) => {
+	cache.set(key, value);
+	if (cache.size <= MAX_SEMANTIC_PIPELINE_CACHE_ENTRIES) return;
+	const oldestKey = cache.keys().next().value;
+	if (oldestKey !== void 0) cache.delete(oldestKey);
+};
+const loadSemanticDictionaries = (key, config) => {
+	const cached = getCachedEntry(dictionaryCache, key);
+	if (cached !== void 0) return cached;
+	let dictionaries;
+	dictionaries = import("@stll/anonymize-data/cities").then(({ loadDictionaryBundle }) => loadDictionaryBundle(defaultDictionaryBundleOptions(config))).catch((error) => {
+		if (dictionaryCache.get(key) === dictionaries) dictionaryCache.delete(key);
+		throw error;
+	});
+	setCachedEntry(dictionaryCache, key, dictionaries);
+	return dictionaries;
+};
+const pipelineConfigFor = (selection) => {
+	if (selection.type === "all") return {
+		...DEFAULT_NATIVE_PIPELINE_CONFIG,
+		labels: [...DEFAULT_NATIVE_PIPELINE_CONFIG.labels]
+	};
+	const [language, ...languages] = selection.languages;
+	return applyPipelineLanguageScope({
+		...DEFAULT_NATIVE_PIPELINE_CONFIG,
+		labels: [...DEFAULT_NATIVE_PIPELINE_CONFIG.labels],
+		workspaceId: `default-pipeline:${pipelineLanguageSelectionKey(selection)}`,
+		...languages.length === 0 ? { language } : { languages: [language, ...languages] }
+	});
+};
+const semanticPipelineCacheFor = (binding) => {
+	const cached = semanticPipelineCache.get(binding);
+	if (cached !== void 0) return cached;
+	const created = /* @__PURE__ */ new Map();
+	semanticPipelineCache.set(binding, created);
+	return created;
+};
+const createSemanticPipeline = ({ binding, selection }) => {
+	const key = pipelineLanguageSelectionKey(selection);
+	const cache = semanticPipelineCacheFor(binding);
+	const cached = getCachedEntry(cache, key);
+	if (cached !== void 0) return cached;
+	const config = pipelineConfigFor(selection);
+	let pipeline;
+	pipeline = loadSemanticDictionaries(key, config).then((dictionaries) => createNativePipelineFromConfig({
+		binding,
+		config: {
+			...config,
+			dictionaries
+		}
+	})).catch((error) => {
+		if (cache.get(key) === pipeline) cache.delete(key);
+		throw error;
+	});
+	setCachedEntry(cache, key, pipeline);
+	return pipeline;
+};
+//#endregion
 //#region src/native-node.ts
 const DEFAULT_NATIVE_PIPELINE_WARMUPS = {
 	lazyRegex: "lazy-regex",
@@ -482,10 +437,11 @@ const convert_external_detection_batch = (document, batch, options = {}) => conv
 	batch
 });
 const normalize_for_search = (text, options = {}) => {
-	return normalize_for_search$1({
+	const args = {
 		binding: resolveNativeSdkBinding(options),
 		text
-	});
+	};
+	return normalize_for_search$1(args);
 };
 const prepare_search_package = (config, { compressed = false, ...options } = {}) => prepare_search_package$1({
 	binding: resolveNativeSdkBinding(options),
@@ -595,6 +551,26 @@ const getDefaultNativePipeline = (options = {}) => {
 	return applyDefaultNativePipelineWarmup(pipeline, resolvedOptions.warmup);
 };
 const get_default_native_pipeline = (options = {}) => getDefaultNativePipeline(options);
+const createPipeline = async ({ language, warmup, ...bindingOptions } = {}) => {
+	const selection = normalizePipelineLanguageSelection(language);
+	if (selection.type === "all") return getDefaultNativePipeline({
+		...bindingOptions,
+		...warmup !== void 0 ? { warmup } : {}
+	});
+	const [singleLanguage, ...additionalLanguages] = selection.languages;
+	if (additionalLanguages.length === 0 && existsSync(defaultNativePipelineLanguagePackageUrl(singleLanguage))) return getDefaultNativePipeline({
+		...bindingOptions,
+		language: singleLanguage,
+		...warmup !== void 0 ? { warmup } : {}
+	});
+	const resolvedWarmup = normalizeDefaultNativePipelineWarmup(warmup);
+	const pipeline = await createSemanticPipeline({
+		binding: resolveNativeSdkBinding(bindingOptions),
+		selection
+	});
+	return applyDefaultNativePipelineWarmup(pipeline, resolvedWarmup);
+};
+const create_pipeline = createPipeline;
 const preloadDefaultNativePipeline = (options = {}) => {
 	const pipeline = getDefaultNativePipeline(options);
 	return applyDefaultNativePipelineWarmup(pipeline, DEFAULT_NATIVE_PIPELINE_WARMUPS.lazyRegex);
@@ -702,10 +678,12 @@ const defaultNativePipelinePackageUrl = (language) => {
 const defaultNativePipelineLanguagePackageUrl = (language) => new URL(`../native-pipeline.${language}.stlanonpkg`, import.meta.url);
 const resolveDefaultNativePipelineLanguage = (language) => {
 	const normalized = normalizeDefaultNativePipelineLanguage(language);
-	if (existsSync(defaultNativePipelineLanguagePackageUrl(normalized))) return normalized;
+	const exactUrl = defaultNativePipelineLanguagePackageUrl(normalized);
+	if (existsSync(exactUrl)) return normalized;
 	const baseLanguage = normalized.split("-").at(0);
 	if (baseLanguage === void 0 || baseLanguage === normalized) return normalized;
-	if (existsSync(defaultNativePipelineLanguagePackageUrl(baseLanguage))) return baseLanguage;
+	const baseUrl = defaultNativePipelineLanguagePackageUrl(baseLanguage);
+	if (existsSync(baseUrl)) return baseLanguage;
 	return normalized;
 };
 const defaultNativePipelinePackageDescription = (language) => language === void 0 ? "Default native pipeline package" : `Default native pipeline package for language "${resolveDefaultNativePipelineLanguage(language)}"`;
@@ -797,6 +775,6 @@ const formatLoadError = (error) => {
 	return String(error);
 };
 //#endregion
-export { redact_text as A, prepareNativePipelinePackage as B, readNativePipelinePackageFile as C, redactDefaultTextJson as D, redactDefaultText as E, DEFAULT_NATIVE_PIPELINE_CONFIG as F, assertNativePipelineSupported as I, createNativePipelineFromConfig as L, redact_text_stream_json as M, setNativeBindingOverride as N, redact_default_text as O, summary_diagnostics_json as P, getNativePipelineCompatibility as R, readDefaultNativePipelinePackageFileAsync as S, read_default_native_pipeline_package_file as T, createPipelineContext as V, preloadDefaultNativePipeline as _, createNativePipelineFromDefaultPackage as a, prepare_search_package as b, diagnostics_json as c, get_default_native_pipeline as d, loadNativeAnonymizeBinding as f, normalize_for_search as g, native_package_version as h, convert_external_detection_batch as i, redact_text_json as j, redact_default_text_json as k, diagnostics_stream_json as l, load_prepared_package_file as m, availableDefaultNativePipelineLanguages as n, createNativePipelineFromPackageFile as o, load_prepared_package as p, available_default_native_pipeline_languages as r, create_native_pipeline_from_default_package as s, DEFAULT_NATIVE_PIPELINE_WARMUPS as t, getDefaultNativePipeline as u, preloadDefaultNativePipelineAsync as v, readNativePipelinePackageFileAsync as w, readDefaultNativePipelinePackageFile as x, preload_default_native_pipeline as y, prepareNativePipelineConfig as z };
+export { redact_default_text as A, createNativePipelineFromConfig as B, readDefaultNativePipelinePackageFile as C, read_default_native_pipeline_package_file as D, readNativePipelinePackageFileAsync as E, setNativeBindingOverride as F, prepareNativePipelineConfig as H, summary_diagnostics_json as I, SUPPORTED_LANGUAGES as L, redact_text as M, redact_text_json as N, redactDefaultText as O, redact_text_stream_json as P, DEFAULT_NATIVE_PIPELINE_CONFIG as R, prepare_search_package as S, readNativePipelinePackageFile as T, prepareNativePipelinePackage as U, getNativePipelineCompatibility as V, createPipelineContext as W, native_package_version as _, createNativePipelineFromDefaultPackage as a, preloadDefaultNativePipelineAsync as b, create_native_pipeline_from_default_package as c, diagnostics_stream_json as d, getDefaultNativePipeline as f, load_prepared_package_file as g, load_prepared_package as h, convert_external_detection_batch as i, redact_default_text_json as j, redactDefaultTextJson as k, create_pipeline as l, loadNativeAnonymizeBinding as m, availableDefaultNativePipelineLanguages as n, createNativePipelineFromPackageFile as o, get_default_native_pipeline as p, available_default_native_pipeline_languages as r, createPipeline as s, DEFAULT_NATIVE_PIPELINE_WARMUPS as t, diagnostics_json as u, normalize_for_search as v, readDefaultNativePipelinePackageFileAsync as w, preload_default_native_pipeline as x, preloadDefaultNativePipeline as y, assertNativePipelineSupported as z };
 
 //# sourceMappingURL=native-node2.mjs.map