@stll/anonymize-wasm 1.4.8 → 1.4.10

package/dist/names-nw-excluded-allcaps.mjs

@@ -0,0 +1,112 @@
+//#region src/data/names-nw-excluded-allcaps.json
+var _comment = "English all-caps words/acronyms excluded from Japanese Hepburn matches";
+var words = [
+	"ADD",
+	"ALL",
+	"AND",
+	"ANY",
+	"API",
+	"APP",
+	"APR",
+	"ARE",
+	"AS",
+	"AT",
+	"AUG",
+	"BIZ",
+	"BUT",
+	"BY",
+	"CAN",
+	"COM",
+	"CSS",
+	"CSV",
+	"DEC",
+	"DEL",
+	"DOC",
+	"EDU",
+	"END",
+	"EUR",
+	"FEB",
+	"FOR",
+	"FRI",
+	"GBP",
+	"GET",
+	"GMT",
+	"GOV",
+	"HAD",
+	"HE",
+	"HH",
+	"HAS",
+	"HTML",
+	"ID",
+	"IF",
+	"IN",
+	"INC",
+	"INFO",
+	"INT",
+	"ISO",
+	"ITS",
+	"JAN",
+	"JUL",
+	"JUN",
+	"KEY",
+	"LLC",
+	"LTD",
+	"MAR",
+	"MAY",
+	"MIL",
+	"MON",
+	"NAME",
+	"NET",
+	"NEW",
+	"NO",
+	"NOT",
+	"NOV",
+	"OCT",
+	"OF",
+	"ON",
+	"ONE",
+	"OR",
+	"ORG",
+	"OUT",
+	"PDF",
+	"POST",
+	"PUT",
+	"RFC",
+	"RUN",
+	"SAT",
+	"SEE",
+	"SEP",
+	"SET",
+	"SQL",
+	"SUN",
+	"TAR",
+	"THE",
+	"THU",
+	"TO",
+	"TUE",
+	"TWO",
+	"TXT",
+	"UID",
+	"URI",
+	"URL",
+	"USA",
+	"USD",
+	"UTC",
+	"VAL",
+	"VAT",
+	"WAS",
+	"WED",
+	"WERE",
+	"XLS",
+	"XML",
+	"YES",
+	"ZIP"
+];
+var names_nw_excluded_allcaps_default = {
+	_comment,
+	words
+};
+//#endregion
+export { _comment, names_nw_excluded_allcaps_default as default, words };
+
+//# sourceMappingURL=names-nw-excluded-allcaps.mjs.map

package/dist/names-nw-excluded-allcaps.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"names-nw-excluded-allcaps.mjs","names":[],"sources":["../../src/data/names-nw-excluded-allcaps.json"],"sourcesContent":[""],"mappings":""}

package/dist/wasm.d.mts

@@ -5,18 +5,45 @@ import { Tokenizer } from "@huggingface/tokenizers";
 
 //#region src/types.d.ts
 /**
- * A detected PII entity span in the source text.
- * Every detection layer produces these.
+ * Fields shared by every entity span in the source text.
  */
-type Entity = {
+type EntityBase = {
   start: number;
   end: number;
   label: string;
   text: string;
   score: number;
-  source: DetectionSource;
   sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
 };
+/**
+ * A PII entity span found by a primary detection layer
+ * (regex, NER, legal forms, deny list, ...).
+ */
+type DetectedEntity = EntityBase & {
+  source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
+};
+/**
+ * An alias mention of a previously detected entity: a
+ * defined term ("the Seller") or a propagated bare
+ * mention ("Acme" after "Acme Corp.").
+ *
+ * `corefSourceText` is required by construction, so an
+ * alias cannot exist without the link back to its source
+ * entity. Placeholder numbering reads it to give the
+ * alias the same placeholder as the source. The link
+ * travels with the entity instead of living in a
+ * side-channel map that a producer could forget to
+ * write — or that a later pass could clear.
+ */
+type CorefAliasEntity = EntityBase & {
+  source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
+  corefSourceText: string;
+};
+/**
+ * A detected PII entity span in the source text.
+ * Every detection layer produces these.
+ */
+type Entity = DetectedEntity | CorefAliasEntity;
 /**
  * Entity after human review. Extends the base Entity
  * with a review decision.
@@ -253,6 +280,13 @@ type Dictionaries = {
    * Merged with legacy config names at init time.
    */
   surnames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Non-Western name tokens per locale code
+   * (e.g., "in", "ar", "ja-latn", "ko", "zh-latn",
+   * "th", "vi", "fil", "id"). Merged with bundled
+   * names-nw-*.json data at init time.
+   */
+  nonWesternNames?: Readonly<Record<string, readonly string[]>>;
   /**
    * Pre-loaded deny-list dictionaries keyed by
    * dictionary ID (e.g., "courts/CZ", "banks/DE").
@@ -521,6 +555,10 @@ declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: Ga
  * shallow copies (spread). Uses position + label so the
  * key is identical for the original object and any
  * `{ ...entity }` copy produced by mergeAndDedup.
+ *
+ * @deprecated No longer used internally: coref alias
+ *   links travel on the entities themselves
+ *   (`corefSourceText`). Kept for API compatibility.
  */
 declare const corefKey: (e: Entity) => string;
 /**
@@ -539,11 +577,20 @@ type NameCorpusData = {
   firstNames: ReadonlySet<string>;
   surnames: ReadonlySet<string>;
   titleTokens: ReadonlySet<string>;
-  excludedWords: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
+  /** Abbreviation-style titles whose trailing dot is
+   *  part of the title, not a sentence boundary.
+   *  Contains the lowercase, dot-stripped form
+   *  (e.g., "dr", "smt", "atty"). */
+  titleAbbreviations: ReadonlySet<string>;
+  excludedWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
+  nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
+  excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
   firstNamesList: readonly string[];
   surnamesList: readonly string[];
   titlesList: readonly string[];
   excludedList: readonly string[];
+  nonWesternNamesList: readonly string[];
+  excludedAllCapsList: readonly string[];
 };
 /**
  * All cached state for a single pipeline run (or
@@ -582,17 +629,6 @@ type PipelineContext = {
   zoneHeadingPatterns: RegExp[] | null;
   zoneSigningPatterns: RegExp[] | null;
   zoneInitPromise: Promise<void> | null;
-  /**
-   * Maps coreference entities to their source entity
-   * text. Populated by findCoreferenceSpans, consumed
-   * by buildPlaceholderMap for consistent placeholder
-   * numbering across aliases and source entities.
-   *
-   * Keyed by `start:end:label` composite string so
-   * lookups survive shallow copies (e.g. from
-   * mergeAndDedup's spread operator).
-   */
-  corefSourceMap: Map<string, string>;
 };
 /** Create a fresh, empty pipeline context. */
 declare const createPipelineContext: () => PipelineContext;
@@ -663,12 +699,11 @@ declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
  * Placeholder format: [LABEL_N] where LABEL is uppercase
  * and N is a 1-based counter per label.
  *
- * @param ctx Pipeline context. Must be the same instance
- *   passed to `runPipeline` (or `findCoreferenceSpans`)
- *   so coreference placeholder links are preserved.
- *   Defaults to `defaultContext` for single-tenant usage.
+ * @param _ctx Unused. Kept for signature compatibility;
+ *   coref alias links now travel on the entities
+ *   themselves (`corefSourceText`).
  */
-declare const buildPlaceholderMap: (entities: Entity[], ctx?: PipelineContext) => Map<string, string>;
+declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext) => Map<string, string>;
 /**
  * Apply redactions to the source text, replacing each
  * confirmed entity span using the configured operator.
@@ -811,11 +846,14 @@ declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?:
  * character before the start and after the end are NOT
  * word characters (letter/digit).
  *
- * Populates `ctx.corefSourceMap` with entries linking
- * each coref entity to its source entity text, for
- * consistent placeholder numbering.
+ * Each returned alias carries `corefSourceText` linking
+ * it to its source entity text, for consistent
+ * placeholder numbering.
+ *
+ * @param _ctx Unused. Kept for signature compatibility;
+ *   alias links now travel on the entities themselves.
  */
-declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], ctx?: PipelineContext) => Entity[];
+declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ctx?: PipelineContext) => Entity[];
 //#endregion
 //#region src/detectors/org-propagation.d.ts
 /**
@@ -824,10 +862,17 @@ declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], ctx
  * to get the base name, and re-scan the full text for
  * bare mentions of that base name. Returns new entities
  * for occurrences not already covered.
+ *
+ * Propagated mentions are coref aliases: each carries
+ * `corefSourceText` linking it to the full seed entity
+ * text, so placeholder numbering assigns the bare
+ * mention the same placeholder as its source ("Acme"
+ * and "Acme Corp." both become [ORGANIZATION_1]).
  */
 declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
 //#endregion
 //#region src/detectors/names.d.ts
+declare const getNameCorpusNonWesternNames: (ctx?: PipelineContext) => readonly string[];
 /**
  * Load name corpus data from injected dictionaries
  * and legacy config files. Merges all sources.
@@ -841,14 +886,18 @@ declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entit
  *   omitted, only legacy config files are used.
  */
 declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionaries, languages?: readonly string[]) => Promise<void>;
+type NameCorpusDetectionOptions = {
+  mode?: "full" | "supplemental";
+};
 /**
  * Detect person names by looking up tokens against the
  * name corpus, then chaining adjacent name-like tokens.
+ * Handles both Western and non-Western name patterns.
  *
  * Requires initNameCorpus() to have been called first.
  * If not initialized, returns an empty array.
  *
- * Scoring:
+ * Scoring (Western):
  *   TITLE + NAME/SURNAME       → 0.95
  *   NAME + NAME/SURNAME        → 0.9
  *   SURNAME + NAME/SURNAME     → 0.9
@@ -856,8 +905,16 @@ declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionarie
  *   ABBREVIATION + NAME        → 0.7
  *   Standalone NAME            → 0.5 (low confidence)
  *   Standalone SURNAME         → skip (too ambiguous)
+ *
+ * Scoring (non-Western, when chain contains nonWestern tokens):
+ *   TITLE + (nonWestern|CAPITALIZED) → 0.95
+ *   JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
+ *   ARABIC_CONNECTOR + nonWestern     → 0.9
+ *   2+ nonWestern tokens              → 0.9
+ *   nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
+ *   Standalone nonWestern mid-sentence → 0.5
  */
-declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext) => Entity[];
+declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext, options?: NameCorpusDetectionOptions) => Entity[];
 //#endregion
 //#region src/unified-search.d.ts
 type UnifiedResult = {
@@ -1122,5 +1179,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
  */
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
-export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
+export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
 //# sourceMappingURL=wasm.d.mts.map