@stll/anonymize 2.8.2 → 2.9.0

package/dist/native-node2.mjs

@@ -1,4 +1,5 @@
 import { PreparedNativeAnonymizer, PreparedNativePipeline, assertNativeBindingVersion, convert_external_detection_batch as convert_external_detection_batch$1, createNativePipelineFromPackage, diagnostics_json as diagnostics_json$1, diagnostics_stream_json as diagnostics_stream_json$1, isNativeAnonymizeBinding, load_prepared_package as load_prepared_package$1, native_package_version as native_package_version$1, normalize_for_search as normalize_for_search$1, prepare_search_package as prepare_search_package$1, redact_text as redact_text$1, redact_text_json as redact_text_json$1, redact_text_stream_json as redact_text_stream_json$1, summary_diagnostics_json as summary_diagnostics_json$1 } from "./native.mjs";
+import { n as applyPipelineLanguageScope, r as language_scopes_default, t as defaultDictionaryBundleOptions } from "./build-native-package2.mjs";
 import { DEFAULT_ENTITY_LABELS } from "./constants.mjs";
 import { createRequire } from "node:module";
 import { existsSync, readFileSync, readdirSync } from "node:fs";
@@ -44,146 +45,6 @@ const createPipelineContext = () => ({
 */
 const defaultContext = createPipelineContext();
 //#endregion
-//#region src/language-scope.ts
-const scopeData = {
-	_comment: "Default dictionary scopes for content language hints. Lower-level caller config can still override name corpus languages and deny-list countries independently.",
-	languages: {
-		"cs": {
-			"nameCorpusLanguages": ["cs", "sk"],
-			"denyListCountries": ["CZ", "SK"]
-		},
-		"de": {
-			"nameCorpusLanguages": ["de"],
-			"denyListCountries": [
-				"DE",
-				"AT",
-				"CH"
-			]
-		},
-		"en": {
-			"nameCorpusLanguages": ["en"],
-			"denyListCountries": [
-				"US",
-				"GB",
-				"CA",
-				"AU",
-				"IE"
-			]
-		},
-		"es": {
-			"nameCorpusLanguages": ["es"],
-			"denyListCountries": [
-				"ES",
-				"MX",
-				"AR",
-				"CL",
-				"CO",
-				"PE",
-				"EC",
-				"VE",
-				"UY",
-				"PY",
-				"BO",
-				"CR",
-				"PA",
-				"DO",
-				"GT",
-				"HN",
-				"SV",
-				"NI",
-				"CU"
-			]
-		},
-		"fr": {
-			"nameCorpusLanguages": ["fr"],
-			"denyListCountries": [
-				"FR",
-				"BE",
-				"CH",
-				"CA",
-				"LU",
-				"MC"
-			]
-		},
-		"hu": {
-			"nameCorpusLanguages": ["hu"],
-			"denyListCountries": ["HU"]
-		},
-		"it": {
-			"nameCorpusLanguages": ["it"],
-			"denyListCountries": ["IT", "CH"]
-		},
-		"lv": {
-			"nameCorpusLanguages": [],
-			"denyListCountries": ["LV"]
-		},
-		"pl": {
-			"nameCorpusLanguages": ["pl"],
-			"denyListCountries": ["PL"]
-		},
-		"pt-br": {
-			"nameCorpusLanguages": ["pt-br"],
-			"denyListCountries": ["BR"]
-		},
-		"ro": {
-			"nameCorpusLanguages": ["ro"],
-			"denyListCountries": ["RO", "MD"]
-		},
-		"sk": {
-			"nameCorpusLanguages": ["sk", "cs"],
-			"denyListCountries": ["SK", "CZ"]
-		},
-		"sv": {
-			"nameCorpusLanguages": ["sv"],
-			"denyListCountries": ["SE", "FI"]
-		}
-	}
-};
-const normalizeLanguage = (language) => language.trim().toLowerCase();
-const fallbackLanguage = (language) => {
-	const index = language.indexOf("-");
-	return index === -1 ? null : language.slice(0, index);
-};
-const uniquePush = (target, values) => {
-	const seen = new Set(target);
-	for (const value of values) {
-		if (seen.has(value)) continue;
-		seen.add(value);
-		target.push(value);
-	}
-};
-const resolveLanguageScope = (language) => {
-	const normalized = normalizeLanguage(language);
-	if (normalized.length === 0) return null;
-	const exact = scopeData.languages[normalized];
-	if (exact !== void 0) return exact;
-	const fallback = fallbackLanguage(normalized);
-	return fallback === null ? null : scopeData.languages[fallback] ?? null;
-};
-const configuredLanguages = (config) => {
-	if (config.languages !== void 0) return config.languages;
-	return config.language === void 0 ? [] : [config.language];
-};
-const applyPipelineLanguageScope = (config) => {
-	const languages = configuredLanguages(config);
-	if (languages.length === 0) return config;
-	const nameCorpusLanguages = [];
-	const denyListCountries = [];
-	for (const language of languages) {
-		const scope = resolveLanguageScope(language);
-		if (scope === null) continue;
-		uniquePush(nameCorpusLanguages, scope.nameCorpusLanguages ?? []);
-		uniquePush(denyListCountries, scope.denyListCountries ?? []);
-	}
-	const next = {};
-	if (config.nameCorpusLanguages === void 0 && nameCorpusLanguages.length > 0) next.nameCorpusLanguages = nameCorpusLanguages;
-	if (config.denyListCountries === void 0 && denyListCountries.length > 0) next.denyListCountries = denyListCountries;
-	return Object.keys(next).length === 0 ? config : {
-		...config,
-		...next
-	};
-};
-//#endregion
 //#region src/types.ts
 const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
 //#endregion
@@ -395,6 +256,98 @@ const DEFAULT_NATIVE_PIPELINE_CONFIG = {
 	workspaceId: "native-pipeline-default"
 };
 //#endregion
+//#region src/pipeline-language.ts
+const isSupportedLanguage = (language) => Object.hasOwn(language_scopes_default.languages, language);
+const SUPPORTED_LANGUAGES = Object.freeze(Object.keys(language_scopes_default.languages).filter(isSupportedLanguage).toSorted());
+const normalizeLanguage = (language) => {
+	if (typeof language !== "string") throw new TypeError("Pipeline language codes must be strings");
+	const normalized = language.trim().toLowerCase();
+	if (!isSupportedLanguage(normalized)) throw new RangeError(`Unsupported pipeline language ${JSON.stringify(language)}; expected one of: ${SUPPORTED_LANGUAGES.join(", ")}`);
+	return normalized;
+};
+const normalizePipelineLanguageSelection = (selection) => {
+	if (selection === void 0 || typeof selection === "string" && selection.trim().toLowerCase() === "all") return { type: "all" };
+	const requested = Array.isArray(selection) ? selection : [selection];
+	if (requested.length === 0) throw new RangeError("Pipeline language selection must not be empty");
+	const normalized = [...new Set(requested.map(normalizeLanguage))].toSorted();
+	const first = normalized.at(0);
+	if (first === void 0) throw new RangeError("Pipeline language selection must not be empty");
+	return {
+		type: "languages",
+		languages: [first, ...normalized.slice(1)]
+	};
+};
+const pipelineLanguageSelectionKey = (selection) => selection.type === "all" ? "all" : selection.languages.join(",");
+//#endregion
+//#region src/create-pipeline.ts
+const dictionaryCache = /* @__PURE__ */ new Map();
+const semanticPipelineCache = /* @__PURE__ */ new WeakMap();
+const MAX_SEMANTIC_PIPELINE_CACHE_ENTRIES = 8;
+const getCachedEntry = (cache, key) => {
+	const cached = cache.get(key);
+	if (cached === void 0) return;
+	cache.delete(key);
+	cache.set(key, cached);
+	return cached;
+};
+const setCachedEntry = (cache, key, value) => {
+	cache.set(key, value);
+	if (cache.size <= MAX_SEMANTIC_PIPELINE_CACHE_ENTRIES) return;
+	const oldestKey = cache.keys().next().value;
+	if (oldestKey !== void 0) cache.delete(oldestKey);
+};
+const loadSemanticDictionaries = (key, config) => {
+	const cached = getCachedEntry(dictionaryCache, key);
+	if (cached !== void 0) return cached;
+	let dictionaries;
+	dictionaries = import("@stll/anonymize-data/cities").then(({ loadDictionaryBundle }) => loadDictionaryBundle(defaultDictionaryBundleOptions(config))).catch((error) => {
+		if (dictionaryCache.get(key) === dictionaries) dictionaryCache.delete(key);
+		throw error;
+	});
+	setCachedEntry(dictionaryCache, key, dictionaries);
+	return dictionaries;
+};
+const pipelineConfigFor = (selection) => {
+	if (selection.type === "all") return {
+		...DEFAULT_NATIVE_PIPELINE_CONFIG,
+		labels: [...DEFAULT_NATIVE_PIPELINE_CONFIG.labels]
+	};
+	const [language, ...languages] = selection.languages;
+	return applyPipelineLanguageScope({
+		...DEFAULT_NATIVE_PIPELINE_CONFIG,
+		labels: [...DEFAULT_NATIVE_PIPELINE_CONFIG.labels],
+		workspaceId: `default-pipeline:${pipelineLanguageSelectionKey(selection)}`,
+		...languages.length === 0 ? { language } : { languages: [language, ...languages] }
+	});
+};
+const semanticPipelineCacheFor = (binding) => {
+	const cached = semanticPipelineCache.get(binding);
+	if (cached !== void 0) return cached;
+	const created = /* @__PURE__ */ new Map();
+	semanticPipelineCache.set(binding, created);
+	return created;
+};
+const createSemanticPipeline = ({ binding, selection }) => {
+	const key = pipelineLanguageSelectionKey(selection);
+	const cache = semanticPipelineCacheFor(binding);
+	const cached = getCachedEntry(cache, key);
+	if (cached !== void 0) return cached;
+	const config = pipelineConfigFor(selection);
+	let pipeline;
+	pipeline = loadSemanticDictionaries(key, config).then((dictionaries) => createNativePipelineFromConfig({
+		binding,
+		config: {
+			...config,
+			dictionaries
+		}
+	})).catch((error) => {
+		if (cache.get(key) === pipeline) cache.delete(key);
+		throw error;
+	});
+	setCachedEntry(cache, key, pipeline);
+	return pipeline;
+};
+//#endregion
 //#region src/native-node.ts
 const DEFAULT_NATIVE_PIPELINE_WARMUPS = {
 	lazyRegex: "lazy-regex",
@@ -595,6 +548,26 @@ const getDefaultNativePipeline = (options = {}) => {
 	return applyDefaultNativePipelineWarmup(pipeline, resolvedOptions.warmup);
 };
 const get_default_native_pipeline = (options = {}) => getDefaultNativePipeline(options);
+const createPipeline = async ({ language, warmup, ...bindingOptions } = {}) => {
+	const selection = normalizePipelineLanguageSelection(language);
+	if (selection.type === "all") return getDefaultNativePipeline({
+		...bindingOptions,
+		...warmup !== void 0 ? { warmup } : {}
+	});
+	const [singleLanguage, ...additionalLanguages] = selection.languages;
+	if (additionalLanguages.length === 0 && existsSync(defaultNativePipelineLanguagePackageUrl(singleLanguage))) return getDefaultNativePipeline({
+		...bindingOptions,
+		language: singleLanguage,
+		...warmup !== void 0 ? { warmup } : {}
+	});
+	const resolvedWarmup = normalizeDefaultNativePipelineWarmup(warmup);
+	const pipeline = await createSemanticPipeline({
+		binding: resolveNativeSdkBinding(bindingOptions),
+		selection
+	});
+	return applyDefaultNativePipelineWarmup(pipeline, resolvedWarmup);
+};
+const create_pipeline = createPipeline;
 const preloadDefaultNativePipeline = (options = {}) => {
 	const pipeline = getDefaultNativePipeline(options);
 	return applyDefaultNativePipelineWarmup(pipeline, DEFAULT_NATIVE_PIPELINE_WARMUPS.lazyRegex);
@@ -797,6 +770,6 @@ const formatLoadError = (error) => {
 	return String(error);
 };
 //#endregion
-export { redact_text as A, prepareNativePipelinePackage as B, readNativePipelinePackageFile as C, redactDefaultTextJson as D, redactDefaultText as E, DEFAULT_NATIVE_PIPELINE_CONFIG as F, assertNativePipelineSupported as I, createNativePipelineFromConfig as L, redact_text_stream_json as M, setNativeBindingOverride as N, redact_default_text as O, summary_diagnostics_json as P, getNativePipelineCompatibility as R, readDefaultNativePipelinePackageFileAsync as S, read_default_native_pipeline_package_file as T, createPipelineContext as V, preloadDefaultNativePipeline as _, createNativePipelineFromDefaultPackage as a, prepare_search_package as b, diagnostics_json as c, get_default_native_pipeline as d, loadNativeAnonymizeBinding as f, normalize_for_search as g, native_package_version as h, convert_external_detection_batch as i, redact_text_json as j, redact_default_text_json as k, diagnostics_stream_json as l, load_prepared_package_file as m, availableDefaultNativePipelineLanguages as n, createNativePipelineFromPackageFile as o, load_prepared_package as p, available_default_native_pipeline_languages as r, create_native_pipeline_from_default_package as s, DEFAULT_NATIVE_PIPELINE_WARMUPS as t, getDefaultNativePipeline as u, preloadDefaultNativePipelineAsync as v, readNativePipelinePackageFileAsync as w, readDefaultNativePipelinePackageFile as x, preload_default_native_pipeline as y, prepareNativePipelineConfig as z };
+export { redact_default_text as A, createNativePipelineFromConfig as B, readDefaultNativePipelinePackageFile as C, read_default_native_pipeline_package_file as D, readNativePipelinePackageFileAsync as E, setNativeBindingOverride as F, prepareNativePipelineConfig as H, summary_diagnostics_json as I, SUPPORTED_LANGUAGES as L, redact_text as M, redact_text_json as N, redactDefaultText as O, redact_text_stream_json as P, DEFAULT_NATIVE_PIPELINE_CONFIG as R, prepare_search_package as S, readNativePipelinePackageFile as T, prepareNativePipelinePackage as U, getNativePipelineCompatibility as V, createPipelineContext as W, native_package_version as _, createNativePipelineFromDefaultPackage as a, preloadDefaultNativePipelineAsync as b, create_native_pipeline_from_default_package as c, diagnostics_stream_json as d, getDefaultNativePipeline as f, load_prepared_package_file as g, load_prepared_package as h, convert_external_detection_batch as i, redact_default_text_json as j, redactDefaultTextJson as k, create_pipeline as l, loadNativeAnonymizeBinding as m, availableDefaultNativePipelineLanguages as n, createNativePipelineFromPackageFile as o, get_default_native_pipeline as p, available_default_native_pipeline_languages as r, createPipeline as s, DEFAULT_NATIVE_PIPELINE_WARMUPS as t, diagnostics_json as u, normalize_for_search as v, readDefaultNativePipelinePackageFileAsync as w, preload_default_native_pipeline as x, preloadDefaultNativePipeline as y, assertNativePipelineSupported as z };
 
 //# sourceMappingURL=native-node2.mjs.map