@stll/anonymize 1.5.0 → 2.0.0-alpha.1

package/scripts/build-native-pipeline-package.mjs

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import {
+  DEFAULT_NATIVE_PIPELINE_CONFIG,
+  prepareNativePipelinePackage,
+} from "../dist/index.mjs";
+import { loadNativeAnonymizeBinding } from "../dist/native-node.mjs";
+
+const args = parseArgs(process.argv.slice(2));
+const outputPath = resolve(args.out ?? "native-pipeline.stlanonpkg");
+const compressed = args.compressed === true;
+const { config, gazetteerEntries } = await loadPackageInput(args);
+const binding = loadNativeAnonymizeBinding();
+const packageBytes = await prepareNativePipelinePackage({
+  binding,
+  config,
+  gazetteerEntries,
+  compressed,
+});
+
+mkdirSync(dirname(outputPath), { recursive: true });
+writeFileSync(outputPath, packageBytes);
+
+console.log(
+  JSON.stringify({
+    event: "native-pipeline-package",
+    outputPath,
+    bytes: packageBytes.byteLength,
+    compressed,
+    nativeVersion: binding.nativePackageVersion(),
+  }),
+);
+
+function parseArgs(values) {
+  const result = {};
+  for (let index = 0; index < values.length; index += 1) {
+    const value = values[index];
+    switch (value) {
+      case "--config": {
+        result.config = requiredValue(values, index, value);
+        index += 1;
+        break;
+      }
+      case "--export": {
+        result.exportName = requiredValue(values, index, value);
+        index += 1;
+        break;
+      }
+      case "--out": {
+        result.out = requiredValue(values, index, value);
+        index += 1;
+        break;
+      }
+      case "--raw": {
+        result.raw = true;
+        break;
+      }
+      case "--compressed": {
+        result.compressed = true;
+        break;
+      }
+      case "--default-dictionaries": {
+        result.defaultDictionaries = true;
+        break;
+      }
+      case "--language": {
+        result.language = requiredValue(values, index, value);
+        index += 1;
+        break;
+      }
+      case "--languages": {
+        result.languages = requiredValue(values, index, value);
+        index += 1;
+        break;
+      }
+      case "--help": {
+        printHelp();
+        process.exit(0);
+      }
+      default:
+        throw new Error(`Unknown option: ${value}`);
+    }
+  }
+  if (result.raw && result.compressed) {
+    throw new Error("Use either --raw or --compressed, not both");
+  }
+  return result;
+}
+
+function requiredValue(values, index, option) {
+  const value = values[index + 1];
+  if (value === undefined || value.startsWith("--")) {
+    throw new Error(`${option} requires a value`);
+  }
+  return value;
+}
+
+async function loadPackageInput(options) {
+  const input = await loadBasePackageInput(options);
+  const withDictionaries =
+    !options.defaultDictionaries || input.config.dictionaries !== undefined
+      ? input
+      : {
+          ...input,
+          config: {
+            ...input.config,
+            dictionaries: await loadDefaultDictionaries(),
+          },
+        };
+  return {
+    ...withDictionaries,
+    config: applyCliLanguageScope(withDictionaries.config, options),
+  };
+}
+
+async function loadBasePackageInput(options) {
+  if (!options.config) {
+    return { config: defaultNativePipelineConfig(), gazetteerEntries: [] };
+  }
+  const moduleUrl = pathToFileURL(resolve(options.config)).href;
+  // eslint-disable-next-line stll/no-dynamic-import-specifier
+  const loaded = await import(moduleUrl);
+  const exportName = options.exportName ?? "default";
+  const candidate =
+    exportName === "default" ? loaded.default : loaded[exportName];
+  if (candidate === undefined) {
+    throw new Error(`Config module does not export ${exportName}`);
+  }
+  const value =
+    typeof candidate === "function" ? await candidate() : await candidate;
+  if (!value || typeof value !== "object") {
+    throw new TypeError("Native package config export must be an object");
+  }
+  if ("config" in value) {
+    return {
+      config: value.config,
+      gazetteerEntries: value.gazetteerEntries ?? [],
+    };
+  }
+  return { config: value, gazetteerEntries: [] };
+}
+
+function defaultNativePipelineConfig() {
+  return {
+    ...DEFAULT_NATIVE_PIPELINE_CONFIG,
+    labels: [...DEFAULT_NATIVE_PIPELINE_CONFIG.labels],
+    workspaceId: "native-pipeline-package",
+  };
+}
+
+function applyCliLanguageScope(pipelineConfig, options) {
+  if (options.language !== undefined && options.languages !== undefined) {
+    throw new Error("Use either --language or --languages, not both");
+  }
+  if (options.language !== undefined) {
+    const language = normalizeLanguageOption(options.language, "--language");
+    return { ...pipelineConfig, language, languages: undefined };
+  }
+  if (options.languages === undefined) {
+    return pipelineConfig;
+  }
+  const languages = normalizeLanguageList(options.languages);
+  return { ...pipelineConfig, language: undefined, languages };
+}
+
+function normalizeLanguageOption(value, option) {
+  const language = value.trim().toLowerCase();
+  if (language.length === 0) {
+    throw new Error(`${option} requires a non-empty language code`);
+  }
+  return language;
+}
+
+function normalizeLanguageList(value) {
+  const languages = value
+    .split(",")
+    .map((entry) => normalizeLanguageOption(entry, "--languages"))
+    .filter((entry, index, entries) => entries.indexOf(entry) === index);
+  if (languages.length === 0) {
+    throw new Error("--languages requires at least one language code");
+  }
+  return languages;
+}
+
+async function loadDefaultDictionaries() {
+  let loaded;
+  try {
+    loaded = await import("@stll/anonymize-data/dictionaries");
+  } catch (error) {
+    throw new Error(
+      `--default-dictionaries requires @stll/anonymize-data: ${formatError(error)}`,
+    );
+  }
+  if (typeof loaded.loadDictionaryBundle !== "function") {
+    throw new TypeError(
+      "@stll/anonymize-data/dictionaries does not export loadDictionaryBundle",
+    );
+  }
+  return loaded.loadDictionaryBundle();
+}
+
+function formatError(error) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return String(error);
+}
+
+function printHelp() {
+  console.log(`Usage: build-native-pipeline-package [options]
+
+Options:
+  --out <path>              Output package path. Defaults to native-pipeline.stlanonpkg.
+  --config <path>           ESM module exporting a PipelineConfig or { config, gazetteerEntries }.
+  --export <name>           Export name to read from the config module. Defaults to default.
+  --language <code>         Build a package scoped to one content language.
+  --languages <codes>       Build a package scoped to comma-separated content languages.
+  --default-dictionaries    Load @stll/anonymize-data into configs that do not provide dictionaries.
+  --compressed              Write an LZ4-compressed package.
+  --raw                     Write an uncompressed package. This is the default.
+`);
+}

package/dist/address-prepositions.mjs

@@ -1,182 +0,0 @@
-//#region src/data/address-prepositions.json
-var _comment = "Prepositions used in street/address names by language. Used by backward scan to continue through lowercase words, and by temporal guard to skip date-like patterns.";
-var address = {
-	"cs": [
-		"nad",
-		"pod",
-		"u",
-		"na",
-		"v",
-		"ve",
-		"ke",
-		"za",
-		"při",
-		"do",
-		"od",
-		"mezi"
-	],
-	"sk": [
-		"nad",
-		"pod",
-		"pri",
-		"na",
-		"vo",
-		"ke",
-		"za",
-		"do",
-		"od",
-		"medzi"
-	],
-	"de": [
-		"am",
-		"an",
-		"im",
-		"bei",
-		"zum",
-		"zur",
-		"auf",
-		"in",
-		"vor"
-	],
-	"en": [
-		"upon",
-		"on",
-		"in",
-		"by",
-		"at",
-		"near"
-	],
-	"fr": [
-		"sur",
-		"sous",
-		"de",
-		"du",
-		"des",
-		"aux",
-		"en"
-	],
-	"pl": [
-		"nad",
-		"pod",
-		"przy",
-		"na",
-		"za",
-		"do",
-		"od",
-		"między"
-	],
-	"it": [
-		"di",
-		"del",
-		"della",
-		"sul",
-		"al"
-	],
-	"es": [
-		"de",
-		"del",
-		"en",
-		"sobre"
-	],
-	"hu": [
-		"melletti",
-		"alatti",
-		"feletti"
-	],
-	"ro": [
-		"de",
-		"din",
-		"pe",
-		"la"
-	],
-	"sv": [
-		"vid",
-		"på",
-		"i"
-	]
-};
-var temporal = {
-	"_comment": "Prepositions that indicate temporal context (dates), not addresses. Used as negative guard.",
-	"cs": [
-		"od",
-		"do",
-		"ke",
-		"na",
-		"dne",
-		"ze",
-		"ve",
-		"za",
-		"při",
-		"mezi"
-	],
-	"sk": [
-		"od",
-		"do",
-		"dňa",
-		"na",
-		"vo",
-		"za"
-	],
-	"de": [
-		"ab",
-		"bis",
-		"am",
-		"vom",
-		"seit"
-	],
-	"en": [
-		"from",
-		"to",
-		"on",
-		"at",
-		"since",
-		"until",
-		"by"
-	],
-	"fr": [
-		"du",
-		"au",
-		"dès",
-		"depuis",
-		"le"
-	],
-	"pl": [
-		"od",
-		"do",
-		"dnia",
-		"na",
-		"za"
-	],
-	"it": [
-		"dal",
-		"al",
-		"fino",
-		"da"
-	],
-	"es": [
-		"del",
-		"al",
-		"desde",
-		"hasta"
-	],
-	"hu": ["tól", "ig"],
-	"ro": [
-		"de",
-		"din",
-		"până"
-	],
-	"sv": [
-		"från",
-		"till",
-		"den"
-	]
-};
-var address_prepositions_default = {
-	_comment,
-	address,
-	temporal
-};
-//#endregion
-export { _comment, address, address_prepositions_default as default, temporal };
-
-//# sourceMappingURL=address-prepositions.mjs.map

package/dist/address-prepositions.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"address-prepositions.mjs","names":[],"sources":["../src/data/address-prepositions.json"],"sourcesContent":[""],"mappings":""}