@stll/anonymize-wasm 1.4.1 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/wasm.mjs CHANGED
@@ -2,7 +2,7 @@ import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_T
2
2
  import { t as amount_words_default } from "./amount-words.mjs";
3
3
  import { t as address_street_types_default } from "./address-street-types.mjs";
4
4
  import { TextSearch } from "@stll/text-search-wasm";
5
- import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
5
+ import { at, au, be, bg, br, ch, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
6
6
  import { toRegex } from "@stll/stdnum/patterns";
7
7
  //#region src/search-engine.ts
8
8
  let _TextSearch;
@@ -31,6 +31,7 @@ const createPipelineContext = () => ({
31
31
  searchKey: "",
32
32
  searchPromise: null,
33
33
  nameCorpus: null,
34
+ nameCorpusKey: "",
34
35
  nameCorpusPromise: null,
35
36
  stopwords: null,
36
37
  stopwordsPromise: null,
@@ -889,7 +890,44 @@ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùû
889
890
  const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
890
891
  const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
891
892
  const ALLCAP_WORD = `[${UPPER}]{2,}`;
893
+ const HSPACE = "(?:[^\\S\\n]|[\xA0 ])";
894
+ const LEGAL_LIST_BOUNDARY_RE = new RegExp(`^[,;]${HSPACE}+(?=\\p{Lu}|(?:\\p{Lu}\\.${HSPACE}?){2,})`, "u");
892
895
  const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
896
+ const EMPTY_LEADING_CLAUSE_TRIMS = {
897
+ phrases: [],
898
+ directPrefixes: []
899
+ };
900
+ let leadingClauseTrimsCache = null;
901
+ let leadingClauseTrimsPromise = null;
902
+ const loadLeadingClauseTrims = async () => {
903
+ if (leadingClauseTrimsCache) return leadingClauseTrimsCache;
904
+ if (leadingClauseTrimsPromise) return leadingClauseTrimsPromise;
905
+ leadingClauseTrimsPromise = (async () => {
906
+ let data = {};
907
+ try {
908
+ const mod = await import("./legal-form-leading-clauses.mjs");
909
+ data = mod.default ?? mod;
910
+ } catch (err) {
911
+ console.warn("[anonymize] legal-forms: failed to load legal-form-leading-clauses.json:", err);
912
+ }
913
+ const phrases = /* @__PURE__ */ new Set();
914
+ const directPrefixes = /* @__PURE__ */ new Set();
915
+ for (const [key, value] of Object.entries(data)) {
916
+ if (key.startsWith("_") || typeof value !== "object" || value === null) continue;
917
+ const config = value;
918
+ for (const phrase of config.phrases ?? []) if (typeof phrase === "string" && phrase.length > 0) phrases.add(phrase);
919
+ for (const prefix of config.directPrefixes ?? []) if (typeof prefix === "string" && prefix.length > 0) directPrefixes.add(prefix);
920
+ }
921
+ const result = {
922
+ phrases: [...phrases],
923
+ directPrefixes: [...directPrefixes]
924
+ };
925
+ leadingClauseTrimsCache = result;
926
+ return result;
927
+ })();
928
+ return leadingClauseTrimsPromise;
929
+ };
930
+ const getLeadingClauseTrimsSync = () => leadingClauseTrimsCache ?? EMPTY_LEADING_CLAUSE_TRIMS;
893
931
  let legalRoleHeadsCache = null;
894
932
  let legalRoleHeadsPromise = null;
895
933
  const loadLegalRoleHeads = async () => {
@@ -917,7 +955,8 @@ const warmLegalRoleHeads = async () => {
917
955
  loadSentenceVerbIndicators(),
918
956
  loadClauseNounHeads(),
919
957
  loadConnectorProseHeads(),
920
- loadStructuralSingleCapPrefixes()
958
+ loadStructuralSingleCapPrefixes(),
959
+ loadLeadingClauseTrims()
921
960
  ]);
922
961
  };
923
962
  let allLegalSuffixesCache = null;
@@ -1054,13 +1093,12 @@ const loadStructuralSingleCapPrefixes = async () => {
1054
1093
  return structuralSingleCapPrefixesPromise;
1055
1094
  };
1056
1095
  const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
1057
- const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
1096
+ const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, `${HSPACE}+`).replace(/\\\./g, `\\.${HSPACE}?`);
1058
1097
  const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1059
1098
  const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1060
1099
  const buildPatternString = (forms) => {
1061
1100
  if (forms.length === 0) return null;
1062
1101
  const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1063
- const HSPACE = "[^\\S\\n]";
1064
1102
  const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
1065
1103
  const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
1066
1104
  const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
@@ -1068,7 +1106,7 @@ const buildPatternString = (forms) => {
1068
1106
  const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
1069
1107
  const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
1070
1108
  const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
1071
- return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1109
+ return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}${`(?:${HSPACE}+|,${HSPACE}*)`}(?:${alt})(?![${LOWER}])`;
1072
1110
  };
1073
1111
  /**
1074
1112
  * Build legal form regex pattern strings.
@@ -1098,10 +1136,17 @@ const buildLegalFormPatterns = async () => {
1098
1136
  if (longPattern) patterns.push(longPattern);
1099
1137
  const shortPattern = buildPatternString(allForms.filter(isShortForm));
1100
1138
  if (shortPattern) patterns.push(shortPattern);
1101
- const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1102
1139
  const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1103
- patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
1104
- patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
1140
+ const mixedNameSep = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
1141
+ const capitalizedNoDigitPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}`;
1142
+ patterns.push(`${capitalizedNoDigitPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
1143
+ const allcapMixedPrefix = `(?:${ALLCAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD}|\\d{1,4})){1,6}`;
1144
+ patterns.push(`${allcapMixedPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
1145
+ const dottedLineWrapPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}\\.${HSPACE}*\\n${HSPACE}*`;
1146
+ patterns.push(`${dottedLineWrapPrefix}(?:${allcapAlt})(?![${LOWER}])`);
1147
+ const allcapPrefix = `(?:${ALLCAP_WORD})(?:(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}(?:${ALLCAP_WORD})){0,2}`;
1148
+ patterns.push(`${allcapPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
1149
+ patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
1105
1150
  return patterns;
1106
1151
  };
1107
1152
  const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
@@ -1110,13 +1155,14 @@ const UPPER_LETTER_RE = /^\p{Lu}/u;
1110
1155
  const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
1111
1156
  const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
1112
1157
  const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
1113
- const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1114
- const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:[ \\t]+|,[ \\t]*)`, "u");
1115
- const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
1158
+ const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:${HSPACE}+|,${HSPACE}*)`, "u");
1159
+ const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)${HSPACE}+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:${HSPACE}+|,${HSPACE}*)`, "u");
1116
1160
  const isStructuralSingleCapMatch = (text) => {
1117
1161
  const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
1118
1162
  return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
1119
1163
  };
1164
+ const findLastSuffixSeparator = (text) => Math.max(text.lastIndexOf(" "), text.lastIndexOf(" "), text.lastIndexOf("\xA0"), text.lastIndexOf(" "), text.lastIndexOf(","));
1165
+ const stripDocxSpaces = (text) => text.replace(/[  ]/g, "");
1120
1166
  /**
1121
1167
  * Find the word ending just before `pos` in `text`,
1122
1168
  * skipping any whitespace (not newlines).
@@ -1180,6 +1226,59 @@ const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
1180
1226
  entityText: entityText.slice(cut)
1181
1227
  };
1182
1228
  };
1229
+ const splitEmbeddedLegalFormList = (entityStart, entityText) => {
1230
+ const cuts = [0];
1231
+ for (const suffix of getAllLegalSuffixesSync()) {
1232
+ const suffixClean = suffix.replace(/[.,\s]/g, "");
1233
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1234
+ let fromIndex = 0;
1235
+ while (fromIndex < entityText.length) {
1236
+ const suffixStart = entityText.indexOf(suffix, fromIndex);
1237
+ if (suffixStart === -1) break;
1238
+ fromIndex = suffixStart + suffix.length;
1239
+ const suffixEnd = suffixStart + suffix.length;
1240
+ if (suffixEnd >= entityText.length - 1) continue;
1241
+ const afterSuffix = entityText.slice(suffixEnd);
1242
+ const boundary = LEGAL_LIST_BOUNDARY_RE.exec(afterSuffix);
1243
+ if (boundary === null) continue;
1244
+ const nextStart = suffixEnd + boundary[0].length;
1245
+ const remainder = entityText.slice(nextStart);
1246
+ if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
1247
+ cuts.push(nextStart);
1248
+ }
1249
+ }
1250
+ const uniqueCuts = [...new Set(cuts)].toSorted((a, b) => a - b);
1251
+ if (uniqueCuts.length === 1) return [{
1252
+ entityStart,
1253
+ entityText
1254
+ }];
1255
+ const segments = [];
1256
+ for (let index = 0; index < uniqueCuts.length; index++) {
1257
+ const start = uniqueCuts[index];
1258
+ const end = uniqueCuts[index + 1] ?? entityText.length;
1259
+ if (start === void 0) continue;
1260
+ const segmentText = entityText.slice(start, end).replace(/[,\s;]+$/u, "");
1261
+ if (segmentText.length === 0) continue;
1262
+ segments.push({
1263
+ entityStart: entityStart + start,
1264
+ entityText: segmentText
1265
+ });
1266
+ }
1267
+ return segments;
1268
+ };
1269
+ const hasDisallowedLineBreak = (text) => {
1270
+ for (const match of text.matchAll(/\n/gu)) {
1271
+ const index = match.index;
1272
+ if (index === void 0) continue;
1273
+ const before = text.slice(0, index);
1274
+ const after = text.slice(index + 1);
1275
+ const dottedDesignatorBefore = /\.[^\S\n]*$/u.test(before);
1276
+ const legalSuffixAfter = /^[^\S\n]*(?:\p{Lu}\.[^\S\n]?){1,}\p{Lu}?\.?$/u.test(after);
1277
+ const allCapsSuffixAfter = /^[^\S\n]*\p{Lu}{2,}\.?$/u.test(after);
1278
+ if (!dottedDesignatorBefore || !legalSuffixAfter && !allCapsSuffixAfter) return true;
1279
+ }
1280
+ return false;
1281
+ };
1183
1282
  const hasMiddleInitialBefore = (fullText, pos) => {
1184
1283
  const previousWord = findWordBefore(fullText, pos);
1185
1284
  if (!previousWord) return false;
@@ -1336,7 +1435,29 @@ const extendBackward = (fullText, matchStart, options = {}) => {
1336
1435
  };
1337
1436
  const trimLeadingClause = (text) => {
1338
1437
  let cut = -1;
1339
- for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
1438
+ const trims = getLeadingClauseTrimsSync();
1439
+ const phraseAlternation = trims.phrases.map(escapeForRegex).join("|");
1440
+ if (phraseAlternation.length > 0) {
1441
+ const phraseRe = new RegExp(`(?:^|\\s)(?:${phraseAlternation})${HSPACE}+`, "giu");
1442
+ for (const match of text.matchAll(phraseRe)) cut = Math.max(cut, match.index + match[0].length);
1443
+ }
1444
+ const directPrefixAlternation = trims.directPrefixes.map(escapeForRegex).join("|");
1445
+ if (directPrefixAlternation.length > 0) {
1446
+ const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
1447
+ for (const match of text.matchAll(directPrefixRe)) {
1448
+ const words = text.slice(0, match.index).match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
1449
+ if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
1450
+ }
1451
+ }
1452
+ for (const match of text.matchAll(/,/gu)) {
1453
+ const comma = match.index;
1454
+ if (comma === void 0) continue;
1455
+ const before = text.slice(0, comma);
1456
+ if (!/\d/u.test(before)) continue;
1457
+ const after = text.slice(comma + 1);
1458
+ const leadingWs = after.match(/^\s*/u)?.[0].length ?? 0;
1459
+ if ((after.slice(leadingWs).match(/\p{Lu}[\p{L}\p{M}\p{N}]*/gu) ?? []).length >= 3) cut = Math.max(cut, comma + 1 + leadingWs);
1460
+ }
1340
1461
  if (cut <= 0) return {
1341
1462
  offset: 0,
1342
1463
  text
@@ -1419,7 +1540,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1419
1540
  }
1420
1541
  }
1421
1542
  }
1422
- if (processedText.includes("\n")) continue;
1543
+ if (processedText.includes("\n") && hasDisallowedLineBreak(processedText)) continue;
1423
1544
  let entityStart = processedStart;
1424
1545
  let entityText = processedText;
1425
1546
  if (fullText && !trimmed) {
@@ -1429,49 +1550,54 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1429
1550
  entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
1430
1551
  }
1431
1552
  }
1432
- const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
1433
- entityStart = listTrim.entityStart;
1434
- entityText = listTrim.entityText;
1435
- const clauseTrim = trimLeadingClause(entityText);
1436
- if (clauseTrim.offset > 0) {
1437
- entityStart += clauseTrim.offset;
1438
- entityText = clauseTrim.text;
1439
- }
1440
- const getPrefixInfo = (value) => {
1441
- const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1442
- return {
1443
- prefixEnd,
1444
- prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1445
- };
1446
- };
1447
- let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1448
- let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1449
- if (isAllCapsMatch && fullText) {
1450
- const lineStart = fullText.lastIndexOf("\n", entityStart);
1451
- const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1452
- const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1453
- const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1454
- if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1455
- if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1456
- entityStart = match.start;
1457
- entityText = text;
1458
- ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1459
- isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1553
+ for (const segment of splitEmbeddedLegalFormList(entityStart, entityText)) {
1554
+ entityStart = segment.entityStart;
1555
+ entityText = segment.entityText;
1556
+ const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
1557
+ entityStart = listTrim.entityStart;
1558
+ entityText = listTrim.entityText;
1559
+ const clauseTrim = trimLeadingClause(entityText);
1560
+ if (clauseTrim.offset > 0) {
1561
+ entityStart += clauseTrim.offset;
1562
+ entityText = clauseTrim.text;
1460
1563
  }
1461
- } else if (isAllCapsMatch) continue;
1462
- const lastSpace = entityText.lastIndexOf(" ");
1463
- const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
1464
- const suffixClean = rawSuffix.replace(/[.,]/g, "");
1465
- if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1466
- if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
1467
- results.push({
1468
- start: entityStart,
1469
- end: entityStart + entityText.length,
1470
- label: "organization",
1471
- text: entityText,
1472
- score: .95,
1473
- source: DETECTION_SOURCES.LEGAL_FORM
1474
- });
1564
+ if (entityText.includes("\n") && hasDisallowedLineBreak(entityText)) continue;
1565
+ const getPrefixInfo = (value) => {
1566
+ const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1567
+ return {
1568
+ prefixEnd,
1569
+ prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1570
+ };
1571
+ };
1572
+ let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1573
+ let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1574
+ if (isAllCapsMatch && fullText) {
1575
+ const lineStart = fullText.lastIndexOf("\n", entityStart);
1576
+ const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1577
+ const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1578
+ const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1579
+ if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1580
+ if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1581
+ entityStart = match.start;
1582
+ entityText = text;
1583
+ ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1584
+ isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1585
+ }
1586
+ } else if (isAllCapsMatch) continue;
1587
+ const lastSuffixSeparator = findLastSuffixSeparator(entityText);
1588
+ const rawSuffix = lastSuffixSeparator !== -1 ? entityText.slice(lastSuffixSeparator + 1) : "";
1589
+ const suffixClean = rawSuffix.replace(/[.,]/g, "");
1590
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1591
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(stripDocxSpaces(entityText.slice(0, lastSuffixSeparator !== -1 ? lastSuffixSeparator : entityText.length)))) continue;
1592
+ results.push({
1593
+ start: entityStart,
1594
+ end: entityStart + entityText.length,
1595
+ label: "organization",
1596
+ text: entityText,
1597
+ score: .95,
1598
+ source: DETECTION_SOURCES.LEGAL_FORM
1599
+ });
1600
+ }
1475
1601
  }
1476
1602
  return results;
1477
1603
  };
@@ -1885,20 +2011,31 @@ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList
1885
2011
  * with per-language first names and surnames. When
1886
2012
  * omitted, only legacy config files are used.
1887
2013
  */
1888
- const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1889
- if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
2014
+ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
2015
+ const languageKey = languages?.toSorted().join(",") ?? "*";
2016
+ if (ctx.nameCorpus && ctx.nameCorpusKey === languageKey) return Promise.resolve();
2017
+ if (ctx.nameCorpusPromise && ctx.nameCorpusKey === languageKey) return ctx.nameCorpusPromise;
2018
+ ctx.nameCorpus = null;
2019
+ ctx.nameCorpusKey = languageKey;
1890
2020
  const promise = (async () => {
1891
2021
  try {
1892
- const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
2022
+ const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod, commonWordsMod] = await Promise.all([
1893
2023
  import("./names-first.mjs"),
1894
2024
  import("./names-surnames.mjs"),
1895
2025
  import("./names-title-tokens.mjs"),
1896
- import("./names-exclusions.mjs")
2026
+ import("./names-exclusions.mjs"),
2027
+ import("./common-words-en.mjs")
1897
2028
  ]);
1898
2029
  const firstNames = [...legacyFirstMod.default.names];
1899
- if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
2030
+ if (dictionaries?.firstNames) {
2031
+ const entries = languages === void 0 ? Object.entries(dictionaries.firstNames) : Object.entries(dictionaries.firstNames).filter(([language]) => languages.includes(language));
2032
+ for (const [, names] of entries) for (const name of names) firstNames.push(name);
2033
+ }
1900
2034
  const surnames = [...legacySurnameMod.default.names];
1901
- if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
2035
+ if (dictionaries?.surnames) {
2036
+ const entries = languages === void 0 ? Object.entries(dictionaries.surnames) : Object.entries(dictionaries.surnames).filter(([language]) => languages.includes(language));
2037
+ for (const [, names] of entries) for (const name of names) surnames.push(name);
2038
+ }
1902
2039
  const dedup = (arr) => {
1903
2040
  const seen = /* @__PURE__ */ new Set();
1904
2041
  const result = [];
@@ -1909,8 +2046,9 @@ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1909
2046
  }
1910
2047
  return result;
1911
2048
  };
2049
+ const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
1912
2050
  const dedupFirst = dedup(firstNames);
1913
- const dedupSurnames = dedup(surnames);
2051
+ const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
1914
2052
  const titles = titleMod.default.tokens;
1915
2053
  const exclusions = exclusionMod.default.words;
1916
2054
  ctx.nameCorpus = {
@@ -2391,9 +2529,14 @@ const STDNUM_ENTRIES = [
2391
2529
  toEntry(at.uid, "tax identification number", .95),
2392
2530
  toEntry(at.tin, "tax identification number", .9),
2393
2531
  toEntry(at.businessid, "registration number", .95),
2532
+ toEntry(ch.uid, "registration number", .95),
2533
+ toEntry(au.abn, "tax identification number", .9),
2534
+ toEntry(au.acn, "registration number", .9),
2394
2535
  toEntry(be.vat, "tax identification number", .95),
2395
2536
  toEntry(be.nn, "national identification number", .9),
2396
2537
  toEntry(nl.vat, "tax identification number", .95),
2538
+ toEntry(no.orgnr, "registration number", .9),
2539
+ toEntry(no.mva, "tax identification number", .95),
2397
2540
  toEntry(dk.vat, "tax identification number", .95),
2398
2541
  toEntry(dk.cpr, "national identification number", .9),
2399
2542
  toEntry(fi.vat, "tax identification number", .95),
@@ -2423,6 +2566,7 @@ const STDNUM_ENTRIES = [
2423
2566
  toEntry(cy.vat, "tax identification number", .95),
2424
2567
  toEntry(mt.vat, "tax identification number", .95),
2425
2568
  toEntry(lu.vat, "tax identification number", .95),
2569
+ toEntry(us.ein, "tax identification number", .9),
2426
2570
  toEntry(br.cpf, "tax identification number", .95),
2427
2571
  toEntry(br.cnpj, "tax identification number", .95)
2428
2572
  ].filter((e) => e !== null);
@@ -2535,6 +2679,30 @@ const ES_CIF = {
2535
2679
  score: .95,
2536
2680
  validator: es.cif
2537
2681
  };
2682
+ const AU_ABN_FORMATTED = {
2683
+ pattern: `\\b\\d{2}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
2684
+ label: "tax identification number",
2685
+ score: .95,
2686
+ validator: au.abn
2687
+ };
2688
+ const NO_ORGNR_FORMATTED = {
2689
+ pattern: `\\b\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
2690
+ label: "registration number",
2691
+ score: .9,
2692
+ validator: no.orgnr
2693
+ };
2694
+ const NO_MVA_FORMATTED = {
2695
+ pattern: "\\bNO[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?MVA\\b",
2696
+ label: "tax identification number",
2697
+ score: .95,
2698
+ validator: no.mva
2699
+ };
2700
+ const US_EIN_FORMATTED = {
2701
+ pattern: `\\b\\d{2}${DASH}\\d{7}\\b`,
2702
+ label: "tax identification number",
2703
+ score: .95,
2704
+ validator: us.ein
2705
+ };
2538
2706
  const BR_CPF_FORMATTED = {
2539
2707
  pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
2540
2708
  label: "tax identification number",
@@ -2655,6 +2823,10 @@ const ALL_REGEX_DEFS = [
2655
2823
  ES_DNI,
2656
2824
  ES_NIE,
2657
2825
  ES_CIF,
2826
+ AU_ABN_FORMATTED,
2827
+ NO_ORGNR_FORMATTED,
2828
+ NO_MVA_FORMATTED,
2829
+ US_EIN_FORMATTED,
2658
2830
  BR_CPF_FORMATTED,
2659
2831
  BR_CNPJ_FORMATTED,
2660
2832
  BR_RG_WITH_SSP,
@@ -4150,6 +4322,25 @@ const loadAllowList = (ctx) => {
4150
4322
  const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
4151
4323
  /** Sync accessor — returns empty set before init. */
4152
4324
  const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
4325
+ let commonWordsPromise = null;
4326
+ let commonWordsCache = null;
4327
+ const loadCommonWords = () => {
4328
+ if (commonWordsCache) return Promise.resolve(commonWordsCache);
4329
+ if (commonWordsPromise) return commonWordsPromise;
4330
+ commonWordsPromise = (async () => {
4331
+ try {
4332
+ const mod = await import("./common-words-en.mjs");
4333
+ const set = new Set((mod.default?.words ?? []).map((word) => word.toLowerCase()));
4334
+ commonWordsCache = set;
4335
+ return set;
4336
+ } catch {
4337
+ const empty = /* @__PURE__ */ new Set();
4338
+ commonWordsCache = empty;
4339
+ return empty;
4340
+ }
4341
+ })();
4342
+ return commonWordsPromise;
4343
+ };
4153
4344
  /**
4154
4345
  * Curated dictionary entries that are pure dotted
4155
4346
  * single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
@@ -4381,6 +4572,21 @@ const SENTENCE_STARTER_WORDS = new Set([
4381
4572
  const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
4382
4573
  const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
4383
4574
  const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
4575
+ const getCityEntries = (dictionaries, allowedCountries) => {
4576
+ const byCountry = dictionaries?.citiesByCountry;
4577
+ if (!byCountry) return dictionaries?.cities ?? [];
4578
+ const result = [];
4579
+ const append = (entries) => {
4580
+ if (!entries) return;
4581
+ for (const entry of entries) result.push(entry);
4582
+ };
4583
+ if (allowedCountries === null) {
4584
+ for (const entries of Object.values(byCountry)) append(entries);
4585
+ return result;
4586
+ }
4587
+ for (const country of allowedCountries) append(byCountry[country.toUpperCase()]);
4588
+ return result;
4589
+ };
4384
4590
  /**
4385
4591
  * Resolve which dictionaries to load based on country
4386
4592
  * and category filters, then build the deny list data.
@@ -4392,21 +4598,24 @@ const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^
4392
4598
  * Returns null if no dictionaries are provided.
4393
4599
  */
4394
4600
  const buildDenyList = async (config, ctx = defaultContext) => {
4395
- await initNameCorpus(ctx, config.dictionaries);
4601
+ await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
4396
4602
  await Promise.all([
4397
4603
  loadStopwords(ctx),
4398
4604
  loadAllowList(ctx),
4399
4605
  loadPersonStopwords(ctx),
4400
4606
  loadAddressStopwords(ctx),
4607
+ loadCommonWords(),
4401
4608
  loadStreetTypeRe(),
4402
4609
  loadGenericRoles(ctx)
4403
4610
  ]);
4611
+ const commonWords = await loadCommonWords();
4404
4612
  const dictionaries = config.dictionaries;
4405
4613
  const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
4406
- const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
4407
4614
  const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
4408
- if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
4409
4615
  const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
4616
+ const cityEntries = getCityEntries(dictionaries, allowedCountries);
4617
+ const hasCities = cityEntries.length > 0;
4618
+ if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
4410
4619
  const excluded = config.denyListExcludeCategories;
4411
4620
  const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
4412
4621
  const patternList = [];
@@ -4417,8 +4626,11 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4417
4626
  const addDenyListEntry = (entry, label, source = "deny-list") => {
4418
4627
  const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
4419
4628
  if (normalized.length === 0) return;
4420
- if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
4421
4629
  const lower = normalized.toLowerCase();
4630
+ if (source !== "custom-deny-list" && label !== "address") {
4631
+ if (SINGLE_WORD_RE.test(normalized) && commonWords.has(lower)) return;
4632
+ if (isShortCuratedNoiseAcronym(normalized)) return;
4633
+ }
4422
4634
  const existing = patternIndex.get(lower);
4423
4635
  if (existing !== void 0) {
4424
4636
  if (!labelList[existing].includes(label)) labelList[existing].push(label);
@@ -4435,10 +4647,12 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4435
4647
  if (hasDenyList) {
4436
4648
  const denyListData = dictionaries.denyList;
4437
4649
  const metaData = dictionaries.denyListMeta;
4650
+ const useScopedNameCorpus = config.nameCorpusLanguages !== void 0;
4438
4651
  for (const [id, entries] of Object.entries(denyListData)) {
4439
4652
  const meta = metaData[id];
4440
4653
  if (!meta) continue;
4441
4654
  if (!config.enableNameCorpus && meta.category === "Names") continue;
4655
+ if (useScopedNameCorpus && meta.category === "Names") continue;
4442
4656
  if (excludeCategories.has(meta.category)) continue;
4443
4657
  if (allowedCountries !== null && meta.country !== null) {
4444
4658
  if (!allowedCountries.has(meta.country)) continue;
@@ -4446,7 +4660,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4446
4660
  for (const entry of entries) addDenyListEntry(entry, meta.label);
4447
4661
  }
4448
4662
  }
4449
- if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
4663
+ if (hasCities && !excludeCategories.has("Places")) for (const entry of cityEntries) addDenyListEntry(entry, "address", "city");
4450
4664
  if (hasCustomDenyList) for (const entry of config.customDenyList) {
4451
4665
  addDenyListEntry(entry.value, entry.label, "custom-deny-list");
4452
4666
  for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
@@ -4541,8 +4755,8 @@ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
4541
4755
  * the search instance was built on a different context
4542
4756
  * (e.g. cachedSearch).
4543
4757
  */
4544
- const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
4545
- await initNameCorpus(ctx, dictionaries);
4758
+ const ensureDenyListData = async (ctx = defaultContext, dictionaries, nameCorpusLanguages) => {
4759
+ await initNameCorpus(ctx, dictionaries, nameCorpusLanguages);
4546
4760
  await Promise.all([
4547
4761
  loadStopwords(ctx),
4548
4762
  loadAllowList(ctx),
@@ -4721,7 +4935,7 @@ const extendCityDistricts = (entities, fullText) => {
4721
4935
  entity.text = fullText.slice(entity.start, entity.end);
4722
4936
  }
4723
4937
  const afterDistrict = fullText.slice(entity.end);
4724
- const dashDistrictM = /^[\s]*[-–][\s]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
4938
+ const dashDistrictM = /^[ \t]{1,4}[-–][ \t]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
4725
4939
  if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
4726
4940
  entity.end += dashDistrictM[0].length;
4727
4941
  entity.text = fullText.slice(entity.start, entity.end);
@@ -4962,6 +5176,7 @@ const hasHouseNumberNearStreetWord = (fullText, seed) => {
4962
5176
  const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
4963
5177
  return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
4964
5178
  };
5179
+ const isLowercaseStreetWordInProse = (fullText, seed) => /^\p{Ll}/u.test(seed.text) && /^\s+\p{Ll}/u.test(fullText.slice(seed.end, seed.end + 16)) && !hasHouseNumberNearStreetWord(fullText, seed);
4965
5180
  const getUsZipPlusFourContext = (fullText, start, seeds) => {
4966
5181
  const stateSeed = getUsStateSeedBeforeZip(fullText, start);
4967
5182
  if (stateSeed !== null) return {
@@ -5022,12 +5237,14 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
5022
5237
  for (const match of allMatches) {
5023
5238
  const idx = match.pattern;
5024
5239
  if (idx < sliceStart || idx >= sliceEnd) continue;
5025
- seeds.push({
5240
+ const seed = {
5026
5241
  type: "street-word",
5027
5242
  start: match.start,
5028
5243
  end: match.end,
5029
5244
  text: match.text
5030
- });
5245
+ };
5246
+ if (isLowercaseStreetWordInProse(fullText, seed)) continue;
5247
+ seeds.push(seed);
5031
5248
  }
5032
5249
  for (const e of existingEntities) {
5033
5250
  if (e.label !== "address") continue;
@@ -5166,6 +5383,7 @@ const NON_ADDRESS_LABELS = new Set([
5166
5383
  ]);
5167
5384
  const expandCluster = async (fullText, cluster, existingEntities) => {
5168
5385
  const { start, end } = cluster;
5386
+ const seedTypes = new Set(cluster.seeds.map((seed) => seed.type));
5169
5387
  let leftBound = 0;
5170
5388
  for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
5171
5389
  let leftPos = start;
@@ -5180,6 +5398,10 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
5180
5398
  if (fullText.slice(p + 1, leftPos).includes("\n")) break;
5181
5399
  leftPos = p + 1;
5182
5400
  }
5401
+ if (!(seedTypes.has("street-word") || seedTypes.has("house-number") || seedTypes.has("postal-code") || seedTypes.has("address-trigger"))) return {
5402
+ start: Math.min(leftPos, start),
5403
+ end
5404
+ };
5183
5405
  let rightPos = end;
5184
5406
  const remaining = fullText.slice(rightPos);
5185
5407
  let nearestBoundary = Math.min(remaining.length, 200);
@@ -6246,9 +6468,12 @@ const enforceBoundaryConsistency = (entities, fullText) => {
6246
6468
  //#region src/build-unified-search.ts
6247
6469
  const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
6248
6470
  const ALNUM_RE = /[\p{L}\p{N}]/u;
6471
+ const createAllowedLabelSet$1 = (labels) => labels.length > 0 ? new Set(labels) : null;
6472
+ const labelIsAllowed$1 = (label, allowedLabels) => allowedLabels === null || allowedLabels.has(label);
6249
6473
  const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
6250
6474
  const legalFormsEnabled = isLegalFormsEnabled(config);
6251
- const customRegexes = config.enableRegex ? config.customRegexes ?? [] : [];
6475
+ const allowedLabels = createAllowedLabelSet$1(config.enableHotwordRules === true ? expandLabelsForHotwordRules(config.labels) : config.labels);
6476
+ const customRegexes = config.enableRegex ? (config.customRegexes ?? []).filter((entry) => labelIsAllowed$1(entry.label, allowedLabels)) : [];
6252
6477
  const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
6253
6478
  legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
6254
6479
  config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
@@ -6257,22 +6482,30 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
6257
6482
  }),
6258
6483
  config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
6259
6484
  buildStreetTypePatterns(),
6260
- getCurrencyPatterns(),
6261
- getDatePatterns(),
6262
- getSigningClausePatterns()
6485
+ config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ? getCurrencyPatterns() : Promise.resolve([]),
6486
+ config.enableRegex && labelIsAllowed$1("date", allowedLabels) ? getDatePatterns() : Promise.resolve([]),
6487
+ config.enableRegex && labelIsAllowed$1("address", allowedLabels) ? getSigningClausePatterns() : Promise.resolve([])
6263
6488
  ]);
6264
- const allRegex = [
6265
- ...REGEX_PATTERNS,
6266
- ...currencyPatterns,
6267
- ...datePatterns,
6268
- ...signingPatterns
6269
- ];
6270
- const regexMeta = [
6271
- ...REGEX_META,
6272
- ...currencyPatterns.map(() => CURRENCY_PATTERN_META),
6273
- ...datePatterns.map(() => DATE_PATTERN_META),
6274
- ...signingPatterns.map(() => SIGNING_CLAUSE_META)
6275
- ];
6489
+ const allRegex = [];
6490
+ const regexMeta = [];
6491
+ if (config.enableRegex) for (const [index, pattern] of REGEX_PATTERNS.entries()) {
6492
+ const meta = REGEX_META[index];
6493
+ if (!meta || !labelIsAllowed$1(meta.label, allowedLabels)) continue;
6494
+ allRegex.push(pattern);
6495
+ regexMeta.push(meta);
6496
+ }
6497
+ for (const pattern of currencyPatterns) {
6498
+ allRegex.push(pattern);
6499
+ regexMeta.push(CURRENCY_PATTERN_META);
6500
+ }
6501
+ for (const pattern of datePatterns) {
6502
+ allRegex.push(pattern);
6503
+ regexMeta.push(DATE_PATTERN_META);
6504
+ }
6505
+ for (const pattern of signingPatterns) {
6506
+ allRegex.push(pattern);
6507
+ regexMeta.push(SIGNING_CLAUSE_META);
6508
+ }
6276
6509
  const customRegexMeta = customRegexes.map((entry) => ({
6277
6510
  label: entry.label,
6278
6511
  score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
@@ -6764,7 +6997,7 @@ const configKey = (config, gazetteerEntries) => {
6764
6997
  score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
6765
6998
  })).sort().join("\n") : "";
6766
6999
  const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
6767
- return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
7000
+ return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.nameCorpusLanguages?.toSorted().join(",") ?? ""}:${config.enableRegex}:${config.labels.toSorted().join(",")}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
6768
7001
  };
6769
7002
  /**
6770
7003
  * Get or build a cached search instance. Cache state
@@ -6784,6 +7017,13 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
6784
7017
  return result;
6785
7018
  };
6786
7019
  /**
7020
+ * Pre-build and cache the unified search instance for a
7021
+ * pipeline configuration. Use the same context in
7022
+ * `runPipeline` to reuse the prepared automata without
7023
+ * passing `cachedSearch` around manually.
7024
+ */
7025
+ const preparePipelineSearch = ({ config, gazetteerEntries = [], context }) => getCachedSearch(config, gazetteerEntries, context ?? defaultContext);
7026
+ /**
6787
7027
  * Run the full detection pipeline.
6788
7028
  *
6789
7029
  * Two TextSearch instances scan the text (regex +
@@ -6840,7 +7080,7 @@ const runPipeline = async (options) => {
6840
7080
  warmAddressStopKeywords(),
6841
7081
  hotwordInit
6842
7082
  ]);
6843
- if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
7083
+ if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries, config.nameCorpusLanguages);
6844
7084
  let zones = [];
6845
7085
  if (config.enableZoneClassification && zoneInitOk) {
6846
7086
  zones = classifyZones(fullText, ctx);
@@ -6849,7 +7089,11 @@ const runPipeline = async (options) => {
6849
7089
  checkAbort(signal);
6850
7090
  const hotwordsActive = enableHotwords && hotwordInitOk;
6851
7091
  const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
6852
- const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
7092
+ const searchConfig = hotwordsActive ? {
7093
+ ...config,
7094
+ labels: [...expandLabelsForHotwordRules(config.labels)]
7095
+ } : config;
7096
+ const search = cachedSearch ?? await getCachedSearch(searchConfig, gazetteerEntries, ctx);
6853
7097
  checkAbort(signal);
6854
7098
  const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
6855
7099
  const { slices } = search;
@@ -6869,7 +7113,7 @@ const runPipeline = async (options) => {
6869
7113
  let rawNameCorpusEntities = [];
6870
7114
  let nameCorpusEntities = [];
6871
7115
  if (config.enableNameCorpus && !config.enableDenyList) {
6872
- await initNameCorpus(ctx, config.dictionaries);
7116
+ await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
6873
7117
  checkAbort(signal);
6874
7118
  rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
6875
7119
  nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
@@ -7568,6 +7812,6 @@ const levenshtein = (rawA, rawB) => {
7568
7812
  //#region src/wasm.ts
7569
7813
  initTextSearch(TextSearch);
7570
7814
  //#endregion
7571
- export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
7815
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
7572
7816
 
7573
7817
  //# sourceMappingURL=wasm.mjs.map