@stll/anonymize-wasm 1.2.0 → 1.4.0

package/dist/wasm.d.mts

@@ -1,29 +1,9 @@
+import { a as OPERATOR_TYPES, i as DetectionSource, n as DETECTION_SOURCES, o as OperatorType, r as DETECTOR_PRIORITY, t as DEFAULT_ENTITY_LABELS } from "./constants2.mjs";
 import { Validator } from "@stll/stdnum";
 import { Match, PatternEntry, TextSearch } from "@stll/text-search";
 import { Tokenizer } from "@huggingface/tokenizers";
 
 //#region src/types.d.ts
-/**
- * Source of a detected entity span.
- * Ordered by detection layer in the pipeline.
- */
-declare const DETECTION_SOURCES: {
-  readonly TRIGGER: "trigger";
-  readonly REGEX: "regex";
-  readonly DENY_LIST: "deny-list";
-  readonly LEGAL_FORM: "legal-form";
-  readonly GAZETTEER: "gazetteer";
-  readonly NER: "ner";
-  readonly COREFERENCE: "coreference";
-};
-type DetectionSource = (typeof DETECTION_SOURCES)[keyof typeof DETECTION_SOURCES];
-/**
- * Priority levels for detection sources.
- * Higher = more structurally reliable. Used during
- * overlap resolution so deterministic detectors beat
- * probabilistic ones regardless of raw score.
- */
-declare const DETECTOR_PRIORITY: Record<DetectionSource, number>;
 /**
  * A detected PII entity span in the source text.
  * Every detection layer produces these.
@@ -62,6 +42,16 @@ type GazetteerEntry = {
 /** Extraction strategy — closed discriminated union. */
 type TriggerStrategy = {
   type: "to-next-comma";
+  /**
+   * Optional list of lowercase keywords that terminate
+   * the value scan, in addition to commas/newlines. Useful
+   * for triggers like court names that may continue past
+   * a missing comma into adjacent clause text ("Městským
+   * soudem v Praze dne 1. 1. 2020"); listing `"dne"` here
+   * stops the scan at the date boundary. Matched on a
+   * word-boundary, case-insensitive.
+   */
+  stopWords?: string[];
 } | {
   type: "to-end-of-line";
 } | {
@@ -72,6 +62,19 @@ type TriggerStrategy = {
 } | {
   type: "address";
   maxChars?: number;
+} | {
+  /**
+   * Extract the first regex match in the value text.
+   * Useful for shape-bounded values that follow a
+   * label on the same line as other fields, where
+   * `to-end-of-line` would over-capture. The pattern
+   * is anchored to the start of the (already
+   * leading-whitespace-stripped) value, so use
+   * `(?:.*?)` prefix only when intentional.
+   */
+  type: "match-pattern";
+  pattern: string;
+  flags?: string;
 };
 /** Validation rules — closed discriminated union. */
 type TriggerValidation = {
@@ -90,7 +93,21 @@ type TriggerValidation = {
   type: "matches-pattern";
   pattern: string;
   flags?: string;
+}
+/**
+ * Run a named stdnum validator (checksum + length)
+ * against the captured value. Keeps the trigger
+ * path symmetrical with the formatted-regex
+ * detectors so e.g. `CPF nº 00000000000` does not
+ * survive as a tax-ID entity.
+ */
+| {
+  type: "valid-id";
+  validator: ValidIdValidator;
 };
+/** Built-in stdnum validators that can be referenced
+ *  by `valid-id` validations. */
+type ValidIdValidator = "br.cpf" | "br.cnpj";
 /** Auto-generated trigger variants — closed set. */
 type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
 /** V2 trigger config entry (JSON shape). */
@@ -124,6 +141,9 @@ type CompiledValidation = {
 } | {
   type: "matches-pattern";
   re: RegExp;
+} | {
+  type: "valid-id";
+  check: (value: string) => boolean;
 };
 /**
  * Runtime rule — one per trigger string after
@@ -136,12 +156,6 @@ type TriggerRule = {
   validations: CompiledValidation[];
   includeTrigger: boolean;
 };
-/**
- * Anonymisation operator types. Each operator defines
- * how a confirmed entity is replaced in the output.
- */
-declare const OPERATOR_TYPES: readonly ["replace", "redact"];
-type OperatorType = (typeof OPERATOR_TYPES)[number];
 /** Per-label operator selection. Key is the entity label. */
 type OperatorConfig = {
   /** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
@@ -301,16 +315,6 @@ type PipelineConfig = {
    */
   dictionaries?: Dictionaries;
 };
-/**
- * Canonical entity labels used across the pipeline.
- * NER models may use different native labels; the bench
- * NER wrapper maps model output to these canonical names.
- *
- * These labels are ephemeral: entities are regenerated on
- * every pipeline run and never persisted to the database.
- * Renaming a label here requires no migration.
- */
-declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel"];
 //#endregion
 //#region src/detectors/regex.d.ts
 type RegexMeta = {
@@ -502,7 +506,9 @@ type PipelineContext = {
   allowList: ReadonlySet<string> | null;
   allowListPromise: Promise<ReadonlySet<string>> | null;
   personStopwords: ReadonlySet<string> | null;
-  personStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
+  personStopwordsPromise: Promise<ReadonlySet<string>> | null;
+  addressStopwords: ReadonlySet<string> | null;
+  addressStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
   firstNameExclusions: ReadonlySet<string> | null;
   firstNameExclusionCorpusLen: number;
   genericRoles: ReadonlySet<string> | null;
@@ -627,6 +633,7 @@ declare const DEFAULT_OPERATOR_CONFIG: OperatorConfig;
 declare const resolveOperator: (config: OperatorConfig, label: string) => OperatorType;
 //#endregion
 //#region src/detectors/legal-forms.d.ts
+declare const warmLegalRoleHeads: () => Promise<void>;
 /**
  * Build legal form regex pattern strings.
  * Returns an array of regex strings for the unified
@@ -638,6 +645,14 @@ declare const buildLegalFormPatterns: () => Promise<string[]>;
  * Process legal form matches from the unified search.
  * Receives all matches; filters to the legal forms
  * slice via sliceStart/sliceEnd.
+ *
+ * The role-head trimming step reads per-language data from
+ * a cache that `runPipeline` warms via `warmLegalRoleHeads()`
+ * before calling this. Callers that invoke
+ * `processLegalFormMatches` directly (without going through
+ * `runPipeline`) must `await warmLegalRoleHeads()` first;
+ * otherwise the trim falls back to a no-op and sentence-
+ * fragment fixes do not apply.
  */
 declare const processLegalFormMatches: (allMatches: Match[], sliceStart: number, sliceEnd: number, fullText?: string) => Entity[];
 //#endregion
@@ -819,6 +834,8 @@ type CountryCode = NonNullRegion[keyof NonNullRegion][number];
 declare const resolveCountries: (regions?: string[], countries?: string[]) => Set<string> | null;
 //#endregion
 //#region src/filters/false-positives.d.ts
+/** Ensure street-type vocabulary is loaded. */
+declare const initAddressComponents: () => Promise<void>;
 /**
  * Filter out entities that are likely false positives:
  * template placeholders, clause/section numbers,
@@ -1035,5 +1052,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
  */
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
-export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
+export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
 //# sourceMappingURL=wasm.d.mts.map