@stll/anonymize-wasm 1.4.9 → 1.4.10

package/dist/names-nw-excluded-allcaps.mjs

@@ -0,0 +1,112 @@
+//#region src/data/names-nw-excluded-allcaps.json
+var _comment = "English all-caps words/acronyms excluded from Japanese Hepburn matches";
+var words = [
+	"ADD",
+	"ALL",
+	"AND",
+	"ANY",
+	"API",
+	"APP",
+	"APR",
+	"ARE",
+	"AS",
+	"AT",
+	"AUG",
+	"BIZ",
+	"BUT",
+	"BY",
+	"CAN",
+	"COM",
+	"CSS",
+	"CSV",
+	"DEC",
+	"DEL",
+	"DOC",
+	"EDU",
+	"END",
+	"EUR",
+	"FEB",
+	"FOR",
+	"FRI",
+	"GBP",
+	"GET",
+	"GMT",
+	"GOV",
+	"HAD",
+	"HE",
+	"HH",
+	"HAS",
+	"HTML",
+	"ID",
+	"IF",
+	"IN",
+	"INC",
+	"INFO",
+	"INT",
+	"ISO",
+	"ITS",
+	"JAN",
+	"JUL",
+	"JUN",
+	"KEY",
+	"LLC",
+	"LTD",
+	"MAR",
+	"MAY",
+	"MIL",
+	"MON",
+	"NAME",
+	"NET",
+	"NEW",
+	"NO",
+	"NOT",
+	"NOV",
+	"OCT",
+	"OF",
+	"ON",
+	"ONE",
+	"OR",
+	"ORG",
+	"OUT",
+	"PDF",
+	"POST",
+	"PUT",
+	"RFC",
+	"RUN",
+	"SAT",
+	"SEE",
+	"SEP",
+	"SET",
+	"SQL",
+	"SUN",
+	"TAR",
+	"THE",
+	"THU",
+	"TO",
+	"TUE",
+	"TWO",
+	"TXT",
+	"UID",
+	"URI",
+	"URL",
+	"USA",
+	"USD",
+	"UTC",
+	"VAL",
+	"VAT",
+	"WAS",
+	"WED",
+	"WERE",
+	"XLS",
+	"XML",
+	"YES",
+	"ZIP"
+];
+var names_nw_excluded_allcaps_default = {
+	_comment,
+	words
+};
+//#endregion
+export { _comment, names_nw_excluded_allcaps_default as default, words };
+
+//# sourceMappingURL=names-nw-excluded-allcaps.mjs.map

package/dist/names-nw-excluded-allcaps.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"names-nw-excluded-allcaps.mjs","names":[],"sources":["../../src/data/names-nw-excluded-allcaps.json"],"sourcesContent":[""],"mappings":""}

package/dist/wasm.d.mts

@@ -280,6 +280,13 @@ type Dictionaries = {
    * Merged with legacy config names at init time.
    */
   surnames?: Readonly<Record<string, readonly string[]>>;
+  /**
+   * Non-Western name tokens per locale code
+   * (e.g., "in", "ar", "ja-latn", "ko", "zh-latn",
+   * "th", "vi", "fil", "id"). Merged with bundled
+   * names-nw-*.json data at init time.
+   */
+  nonWesternNames?: Readonly<Record<string, readonly string[]>>;
   /**
    * Pre-loaded deny-list dictionaries keyed by
    * dictionary ID (e.g., "courts/CZ", "banks/DE").
@@ -570,11 +577,20 @@ type NameCorpusData = {
   firstNames: ReadonlySet<string>;
   surnames: ReadonlySet<string>;
   titleTokens: ReadonlySet<string>;
-  excludedWords: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
+  /** Abbreviation-style titles whose trailing dot is
+   *  part of the title, not a sentence boundary.
+   *  Contains the lowercase, dot-stripped form
+   *  (e.g., "dr", "smt", "atty"). */
+  titleAbbreviations: ReadonlySet<string>;
+  excludedWords: ReadonlySet<string>; /** Non-Western name tokens merged across all locales. */
+  nonWesternNames: ReadonlySet<string>; /** All-caps acronyms excluded from name detection. */
+  excludedAllCaps: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
   firstNamesList: readonly string[];
   surnamesList: readonly string[];
   titlesList: readonly string[];
   excludedList: readonly string[];
+  nonWesternNamesList: readonly string[];
+  excludedAllCapsList: readonly string[];
 };
 /**
  * All cached state for a single pipeline run (or
@@ -856,6 +872,7 @@ declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ct
 declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
 //#endregion
 //#region src/detectors/names.d.ts
+declare const getNameCorpusNonWesternNames: (ctx?: PipelineContext) => readonly string[];
 /**
  * Load name corpus data from injected dictionaries
  * and legacy config files. Merges all sources.
@@ -869,14 +886,18 @@ declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entit
  *   omitted, only legacy config files are used.
  */
 declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionaries, languages?: readonly string[]) => Promise<void>;
+type NameCorpusDetectionOptions = {
+  mode?: "full" | "supplemental";
+};
 /**
  * Detect person names by looking up tokens against the
  * name corpus, then chaining adjacent name-like tokens.
+ * Handles both Western and non-Western name patterns.
  *
  * Requires initNameCorpus() to have been called first.
  * If not initialized, returns an empty array.
  *
- * Scoring:
+ * Scoring (Western):
  *   TITLE + NAME/SURNAME       → 0.95
  *   NAME + NAME/SURNAME        → 0.9
  *   SURNAME + NAME/SURNAME     → 0.9
@@ -884,8 +905,16 @@ declare const initNameCorpus: (ctx?: PipelineContext, dictionaries?: Dictionarie
  *   ABBREVIATION + NAME        → 0.7
  *   Standalone NAME            → 0.5 (low confidence)
  *   Standalone SURNAME         → skip (too ambiguous)
- */
-declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext) => Entity[];
+ *
+ * Scoring (non-Western, when chain contains nonWestern tokens):
+ *   TITLE + (nonWestern|CAPITALIZED) → 0.95
+ *   JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
+ *   ARABIC_CONNECTOR + nonWestern     → 0.9
+ *   2+ nonWestern tokens              → 0.9
+ *   nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
+ *   Standalone nonWestern mid-sentence → 0.5
+ */
+declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext, options?: NameCorpusDetectionOptions) => Entity[];
 //#endregion
 //#region src/unified-search.d.ts
 type UnifiedResult = {
@@ -1150,5 +1179,5 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
  */
 declare const normalizeForSearch: (text: string) => string;
 //#endregion
-export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
+export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, type CustomDenyListEntry, type CustomRegexPattern, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type Dictionaries, type DictionaryMeta, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, type PipelineSearchOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
 //# sourceMappingURL=wasm.d.mts.map