@stll/anonymize 2.8.2 → 2.9.0

package/dist/types.d.mts

@@ -0,0 +1,429 @@
+import { a as DetectionSource, n as DETECTION_SOURCES, p as OperatorType } from "./constants2.mjs";
+//#region src/types.d.ts
+/**
+ * Fields shared by every entity span in the source text.
+ */
+type EntityBase = {
+  start: number;
+  end: number;
+  label: string;
+  text: string;
+  score: number;
+  sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
+};
+/**
+ * A PII entity span found by a primary detection layer
+ * (regex, NER, legal forms, deny list, ...).
+ */
+type DetectedEntity = EntityBase & {
+  source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
+};
+/**
+ * An alias mention of a previously detected entity: a
+ * defined term ("the Seller") or a propagated bare
+ * mention ("Acme" after "Acme Corp.").
+ *
+ * `corefSourceText` is required by construction, so an
+ * alias cannot exist without the link back to its source
+ * entity. Placeholder numbering reads it to give the
+ * alias the same placeholder as the source. The link
+ * travels with the entity instead of living in a
+ * side-channel map that a producer could forget to
+ * write — or that a later pass could clear.
+ */
+type CorefAliasEntity = EntityBase & {
+  source: typeof DETECTION_SOURCES.COREFERENCE;
+  /** Full text of the source entity this alias refers to. */
+  corefSourceText: string;
+};
+/**
+ * A detected PII entity span in the source text.
+ * Every detection layer produces these.
+ */
+type Entity = DetectedEntity | CorefAliasEntity;
+/**
+ * Entity after human review. Extends the base Entity
+ * with a review decision.
+ */
+type ReviewDecision = "confirmed" | "rejected" | "relabeled";
+type ReviewedEntity = Entity & {
+  decision?: ReviewDecision;
+  originalLabel?: string;
+};
+/**
+ * A single entry in the workspace-scoped gazetteer
+ * (deny list). Persisted in IndexedDB.
+ */
+type GazetteerEntry = {
+  id: string;
+  canonical: string;
+  label: string;
+  variants: string[];
+  workspaceId: string;
+  createdAt: number;
+  source: "manual" | "confirmed-from-model";
+};
+/** Extraction strategy — closed discriminated union. */
+type TriggerStrategy = {
+  type: "to-next-comma";
+  /**
+   * Optional list of lowercase keywords that terminate
+   * the value scan, in addition to commas/newlines. Useful
+   * for triggers like court names that may continue past
+   * a missing comma into adjacent clause text ("Městským
+   * soudem v Praze dne 1. 1. 2020"); listing `"dne"` here
+   * stops the scan at the date boundary. Matched on a
+   * word-boundary, case-insensitive.
+   */
+  stopWords?: string[];
+  /**
+   * Hard cap on the captured span length, in characters,
+   * regardless of where the next comma / stop char sits.
+   * Use for triggers that label short formulaic phrases
+   * ("State of Delaware") and must not absorb the rest
+   * of a long forum-selection clause when the comma is
+   * sentences away. Falls back to the default 100-char
+   * fallback when omitted.
+   */
+  maxLength?: number;
+} | {
+  type: "to-end-of-line";
+} | {
+  type: "n-words";
+  count: number;
+} | {
+  type: "company-id-value";
+} | {
+  type: "address";
+  maxChars?: number;
+} | {
+  /**
+   * Extract the first regex match in the value text.
+   * Useful for shape-bounded values that follow a
+   * label on the same line as other fields, where
+   * `to-end-of-line` would over-capture. The pattern
+   * is anchored to the start of the (already
+   * leading-whitespace-stripped) value, so use
+   * `(?:.*?)` prefix only when intentional.
+   */
+  type: "match-pattern";
+  pattern: string;
+  flags?: string;
+};
+/** Validation rules — closed discriminated union. */
+type TriggerValidation = {
+  type: "starts-uppercase";
+} | {
+  type: "min-length";
+  min: number;
+} | {
+  type: "max-length";
+  max: number;
+} | {
+  type: "no-digits";
+} | {
+  type: "has-digits";
+} | {
+  type: "matches-pattern";
+  pattern: string;
+  flags?: string;
+} |
+/**
+ * Run a named stdnum validator (checksum + length)
+ * against the captured value. Keeps the trigger
+ * path symmetrical with the formatted-regex
+ * detectors so e.g. `CPF nº 00000000000` does not
+ * survive as a tax-ID entity.
+ */
+{
+  type: "valid-id";
+  validator: ValidIdValidator;
+};
+/** Built-in stdnum validators that can be referenced
+ *  by `valid-id` validations. */
+type ValidIdValidator = "br.cpf" | "br.cnpj" | "us.rtn";
+/** Auto-generated trigger variants — closed set. */
+type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
+/** V2 trigger config entry (JSON shape). */
+type TriggerGroupConfig = {
+  id?: string;
+  triggers: string[];
+  label: string;
+  strategy: TriggerStrategy;
+  extensions?: TriggerExtension[];
+  validations?: TriggerValidation[];
+  /** When true, include the trigger text in the
+   *  entity span (e.g., court names). */
+  includeTrigger?: boolean;
+};
+/** Compiled validation with pre-built regex. */
+type CompiledValidation = {
+  type: "starts-uppercase";
+  re: RegExp;
+} | {
+  type: "min-length";
+  min: number;
+} | {
+  type: "max-length";
+  max: number;
+} | {
+  type: "no-digits";
+  re: RegExp;
+} | {
+  type: "has-digits";
+  re: RegExp;
+} | {
+  type: "matches-pattern";
+  re: RegExp;
+} | {
+  type: "valid-id";
+  validator: ValidIdValidator;
+  check: (value: string) => boolean;
+};
+/**
+ * Runtime rule — one per trigger string after
+ * expansion. Fed to the Aho-Corasick automaton.
+ */
+type TriggerRule = {
+  trigger: string;
+  label: string;
+  strategy: TriggerStrategy;
+  validations: CompiledValidation[];
+  includeTrigger: boolean;
+};
+/** Per-label operator selection. Key is the entity label. */
+type MaskDirection = "start" | "end";
+type MaskOperatorConfig = {
+  type: "mask";
+  maskingCharacter: string;
+  charactersToMask: number;
+  direction: MaskDirection;
+};
+type OperatorSelection = Exclude<OperatorType, "mask"> | MaskOperatorConfig;
+type OperatorConfig = {
+  /** Operator per label. Missing labels default to "replace". */
+  operators: Record<string, OperatorSelection>;
+  /** Custom replacement string for the redact operator. */
+  redactString: string;
+};
+/** Whether an operator produces a reversible redaction entry. */
+type OperatorReversibility = "reversible" | "irreversible" | "preserving";
+type AnonymisationOperator = {
+  type: OperatorType;
+  reversibility: OperatorReversibility;
+  /**
+   * Apply the operator to a single entity occurrence.
+   * Returns the replacement string to embed in the document.
+   */
+  apply: (text: string, label: string, placeholder: string, redactString: string, selection: OperatorSelection) => string;
+};
+/**
+ * Redacted document output with stable entity mapping.
+ */
+type RedactionResult = {
+  redactedText: string;
+  /**
+   * Maps placeholder to original text. Only populated for
+   * reversible operators (replace). Empty for redact, keep, and mask.
+   */
+  redactionMap: Map<string, string>;
+  /** Maps placeholder to the operator that produced it. */
+  operatorMap: Map<string, OperatorType>;
+  entityCount: number;
+};
+/**
+ * Configuration for the detection pipeline.
+ */
+type DenyListCategory = "Names" | "Places" | "Addresses" | "Courts" | "Financial" | "Government" | "Healthcare" | "Education" | "Political" | "Organizations" | "International";
+/**
+ * Metadata for a single dictionary entry in the
+ * deny-list system. Mirrors the shape from
+ * the anonymize-data package so consumers can pass
+ * pre-loaded data without a runtime dependency.
+ */
+type DictionaryMeta = {
+  label: string;
+  category: DenyListCategory;
+  country: string | null;
+};
+/**
+ * Caller-supplied exact terms for deny-list matching.
+ * These entries are merged with the published deny-list
+ * dictionaries when `enableDenyList` is enabled.
+ */
+type CustomDenyListEntry = {
+  value: string;
+  label: string;
+  variants?: readonly string[];
+};
+/**
+ * Caller-supplied regex detector. The pattern is passed
+ * to the native Rust regex engine, so use its supported
+ * regex syntax. Inline flags such as `(?i)` are accepted
+ * when supported by that engine.
+ */
+type CustomRegexPattern = {
+  pattern: string;
+  label: string;
+  score?: number;
+  preparedArtifactPolicy?: "include" | "omit";
+};
+/**
+ * Pre-loaded dictionary data for dependency injection.
+ * Consumers that want name/city/deny-list detection
+ * load dictionaries themselves (e.g. from the
+ * anonymize-data package) and pass them here; the
+ * anonymize package has zero cross-package imports.
+ *
+ * All fields are optional. When a field is absent,
+ * the corresponding detection path is skipped (same
+ * behavior as when no dictionaries are available).
+ */
+type Dictionaries = {
+  /**
+   * First names per language code (e.g., "cs", "de").
+   */
+  firstNames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Surnames per language code.
+   */
+  surnames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Non-Western name tokens per locale code
+   * (e.g., "in", "ar", "ja-latn", "ko", "zh-latn",
+   * "th", "vi", "fil", "id"). Merged with bundled
+   * names-nw-*.json data at init time.
+   */
+  nonWesternNames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Pre-loaded deny-list dictionaries keyed by
+   * dictionary ID (e.g., "courts/CZ", "banks/DE").
+   * Each value is the array of terms for that
+   * dictionary.
+   */
+  denyList?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Metadata per dictionary ID. Required when
+   * `denyList` is provided so the pipeline knows
+   * labels, categories, and country filters.
+   */
+  denyListMeta?: Readonly<Record<string, DictionaryMeta>>;
+  /**
+   * Pre-loaded city names, already merged across
+   * all desired countries.
+   *
+   * Prefer `citiesByCountry` when callers also pass
+   * `denyListCountries` / `denyListRegions`; merged
+   * city arrays cannot be scoped after injection.
+   */
+  cities?: readonly string[];
+  /**
+   * Pre-loaded city names keyed by ISO 3166-1 alpha-2
+   * country code. When provided, the deny-list builder
+   * applies `denyListCountries` / `denyListRegions`
+   * before adding city patterns to the search automaton.
+   */
+  citiesByCountry?: Readonly<Record<string, readonly string[]>>;
+};
+/**
+ * Street-address detection without a known-city anchor.
+ */
+type StandaloneStreetDetection = "off" | "houseNumberAnchored";
+type PipelineConfig = {
+  threshold: number;
+  enableTriggerPhrases: boolean;
+  enableRegex: boolean;
+  /**
+   * Expected content language codes. When present, these
+   * derive default dictionary scopes for name corpus and
+   * deny-list matching unless the lower-level scope fields
+   * below are set explicitly.
+   */
+  languages?: string[];
+  /**
+   * Convenience form for single-language documents. Ignored
+   * when `languages` is also provided.
+   */
+  language?: string;
+  /**
+   * Enables legal-form organization detection.
+   * Required for typed callers; legacy untyped
+   * callers that omit this field are treated as
+   * enabled at runtime for backward compatibility.
+   */
+  enableLegalForms: boolean;
+  /**
+   * Enables first-name/surname/title corpus matching.
+   * When deny-list mode is enabled, this also controls
+   * whether name-corpus entries are injected into the
+   * deny-list search automaton.
+   */
+  enableNameCorpus: boolean;
+  /**
+   * Optional language scope for first-name/surname
+   * dictionaries, using the keys present in
+   * `dictionaries.firstNames` / `dictionaries.surnames`
+   * (for example `["en", "de"]`). When omitted, all
+   * injected name languages are used for backward
+   * compatibility.
+   */
+  nameCorpusLanguages?: string[];
+  enableDenyList: boolean;
+  denyListCountries?: string[];
+  denyListRegions?: string[];
+  denyListExcludeCategories?: string[];
+  /**
+   * Caller-owned exact terms to match through the
+   * deny-list layer. Requires `enableDenyList: true`.
+   */
+  customDenyList?: readonly CustomDenyListEntry[];
+  /**
+   * Caller-owned regex detectors. Requires
+   * `enableRegex: true`.
+   */
+  customRegexes?: readonly CustomRegexPattern[];
+  enableGazetteer: boolean;
+  /**
+   * Detect country names (ISO 3166-1 names, curated
+   * aliases, alpha-3 codes). Defaults to true. Names
+   * span all manifest languages plus widely-used
+   * additions (Dutch, Russian, Chinese, Arabic, etc.).
+   */
+  enableCountries?: boolean;
+  enableConfidenceBoost: boolean;
+  enableCoreference: boolean;
+  enableZoneClassification?: boolean;
+  enableHotwordRules?: boolean;
+  /**
+   * Detect a street address that carries no known-city
+   * anchor. Defaults to `"off"`.
+   *
+   * `"houseNumberAnchored"` accepts a street-type word
+   * with a house number directly beside it, in either
+   * order ("14 Rue de la Paix", "Hauptstraße 5",
+   * "123 Main Street"). A bare street name with no
+   * number never fires.
+   *
+   * A street-type word plus a nearby number is a much
+   * weaker signal than a city-anchored address and does
+   * fire on contract prose ("District Court 2019"), so
+   * this stays opt-in per workspace.
+   */
+  standaloneStreetDetection?: StandaloneStreetDetection;
+  /**
+   * Requested output labels. An empty array means
+   * "do not filter by label" for deterministic detectors.
+   */
+  labels: string[];
+  workspaceId: string;
+  /**
+   * Pre-loaded dictionary data for name, deny-list,
+   * and city detection. When omitted, dictionary-based
+   * detection paths are skipped. Consumers load from
+   * the anonymize-data package and pass the data here.
+   */
+  dictionaries?: Dictionaries;
+};
+//#endregion
+export { TriggerRule as _, Dictionaries as a, GazetteerEntry as c, PipelineConfig as d, RedactionResult as f, TriggerGroupConfig as g, TriggerExtension as h, DenyListCategory as i, OperatorConfig as l, ReviewedEntity as m, CustomDenyListEntry as n, DictionaryMeta as o, ReviewDecision as p, CustomRegexPattern as r, Entity as s, AnonymisationOperator as t, OperatorSelection as u, TriggerStrategy as v, TriggerValidation as y };
+//# sourceMappingURL=types.d.mts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stll/anonymize",
-  "version": "2.8.2",
+  "version": "2.9.0",
   "description": "Deterministic PII detection and anonymization with regex, deny lists, and coreference resolution",
   "keywords": [
     "anonymization",
@@ -104,24 +104,18 @@
     "smoke:wasm-browser": "node scripts/smoke-wasm-browser.mjs",
     "format": "oxfmt ."
   },
-  "peerDependencies": {
+  "dependencies": {
     "@stll/anonymize-data": "^0.0.10"
   },
-  "peerDependenciesMeta": {
-    "@stll/anonymize-data": {
-      "optional": true
-    }
-  },
   "optionalDependencies": {
-    "@stll/anonymize-darwin-arm64": "2.8.2",
-    "@stll/anonymize-darwin-x64": "2.8.2",
-    "@stll/anonymize-linux-arm64-gnu": "2.8.2",
-    "@stll/anonymize-linux-x64-gnu": "2.8.2",
-    "@stll/anonymize-win32-x64-msvc": "2.8.2"
+    "@stll/anonymize-darwin-arm64": "2.9.0",
+    "@stll/anonymize-darwin-x64": "2.9.0",
+    "@stll/anonymize-linux-arm64-gnu": "2.9.0",
+    "@stll/anonymize-linux-x64-gnu": "2.9.0",
+    "@stll/anonymize-win32-x64-msvc": "2.9.0"
   },
   "devDependencies": {
     "@napi-rs/cli": "^3.8.6",
-    "@stll/anonymize-data": "workspace:*",
     "bun-types": "1.4.0",
     "fast-check": "^4.9.0",
     "fflate": "^0.8.3",

package/scripts/build-native-pipeline-package.mjs

@@ -6,7 +6,12 @@ import { pathToFileURL } from "node:url";
 import {
   DEFAULT_NATIVE_PIPELINE_CONFIG,
   prepareNativePipelinePackage,
+  SUPPORTED_LANGUAGES,
 } from "../dist/index.mjs";
+import {
+  applyPipelineLanguageScope,
+  defaultDictionaryBundleOptions,
+} from "../dist/build-native-package.mjs";
 import { loadNativeAnonymizeBinding } from "../dist/native-node.mjs";
 
 const args = parseArgs(process.argv.slice(2));
@@ -100,20 +105,20 @@ function requiredValue(values, index, option) {
 
 async function loadPackageInput(options) {
   const input = await loadBasePackageInput(options);
+  const scopedConfig = applyPipelineLanguageScope(
+    applyCliLanguageScope(input.config, options),
+  );
   const withDictionaries =
-    !options.defaultDictionaries || input.config.dictionaries !== undefined
-      ? input
+    !options.defaultDictionaries || scopedConfig.dictionaries !== undefined
+      ? { ...input, config: scopedConfig }
       : {
           ...input,
           config: {
-            ...input.config,
-            dictionaries: await loadDefaultDictionaries(),
+            ...scopedConfig,
+            dictionaries: await loadDefaultDictionaries(scopedConfig),
           },
         };
-  return {
-    ...withDictionaries,
-    config: applyCliLanguageScope(withDictionaries.config, options),
-  };
+  return withDictionaries;
 }
 
 async function loadBasePackageInput(options) {
@@ -171,6 +176,11 @@ function normalizeLanguageOption(value, option) {
   if (language.length === 0) {
     throw new Error(`${option} requires a non-empty language code`);
   }
+  if (!SUPPORTED_LANGUAGES.includes(language)) {
+    throw new Error(
+      `Unsupported pipeline language ${JSON.stringify(value)}; expected one of: ${SUPPORTED_LANGUAGES.join(", ")}`,
+    );
+  }
   return language;
 }
 
@@ -185,7 +195,7 @@ function normalizeLanguageList(value) {
   return languages;
 }
 
-async function loadDefaultDictionaries() {
+async function loadDefaultDictionaries(pipelineConfig) {
   let loaded;
   try {
     // The bundle loads city dictionaries, so it ships in the data package's
@@ -203,7 +213,9 @@ async function loadDefaultDictionaries() {
       "@stll/anonymize-data/cities does not export loadDictionaryBundle",
     );
   }
-  return loaded.loadDictionaryBundle();
+  return loaded.loadDictionaryBundle(
+    defaultDictionaryBundleOptions(pipelineConfig),
+  );
 }
 
 function formatError(error) {