@stll/anonymize-wasm 1.4.1 → 1.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/wasm.mjs CHANGED
@@ -2,7 +2,7 @@ import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_T
2
2
  import { t as amount_words_default } from "./amount-words.mjs";
3
3
  import { t as address_street_types_default } from "./address-street-types.mjs";
4
4
  import { TextSearch } from "@stll/text-search-wasm";
5
- import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
5
+ import { at, au, be, bg, br, ch, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
6
6
  import { toRegex } from "@stll/stdnum/patterns";
7
7
  //#region src/search-engine.ts
8
8
  let _TextSearch;
@@ -31,6 +31,7 @@ const createPipelineContext = () => ({
31
31
  searchKey: "",
32
32
  searchPromise: null,
33
33
  nameCorpus: null,
34
+ nameCorpusKey: "",
34
35
  nameCorpusPromise: null,
35
36
  stopwords: null,
36
37
  stopwordsPromise: null,
@@ -230,12 +231,22 @@ const LEGAL_SUFFIXES = [...[
230
231
  "OHG",
231
232
  "Ltd.",
232
233
  "Ltd",
234
+ "LTD.",
235
+ "LTD",
233
236
  "LLC",
234
237
  "LLP",
235
238
  "Inc.",
239
+ "INC.",
240
+ "Inc",
241
+ "INC",
236
242
  "Corp.",
243
+ "CORP.",
244
+ "Corp",
245
+ "CORP",
237
246
  "Corporation",
247
+ "CORPORATION",
238
248
  "Co.",
249
+ "CO.",
239
250
  "LP",
240
251
  "L.P.",
241
252
  "PLC",
@@ -245,6 +256,8 @@ const LEGAL_SUFFIXES = [...[
245
256
  "B.V.",
246
257
  "Pty Ltd.",
247
258
  "Pty Ltd",
259
+ "PTY LTD.",
260
+ "PTY LTD",
248
261
  "S.A.",
249
262
  "SA",
250
263
  "SAS",
@@ -889,7 +902,44 @@ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùû
889
902
  const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
890
903
  const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
891
904
  const ALLCAP_WORD = `[${UPPER}]{2,}`;
905
+ const HSPACE = "(?:[^\\S\\n]|[\xA0 ])";
906
+ const LEGAL_LIST_BOUNDARY_RE = new RegExp(`^[,;]${HSPACE}+(?=\\p{Lu}|(?:\\p{Lu}\\.${HSPACE}?){2,})`, "u");
892
907
  const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
908
+ const EMPTY_LEADING_CLAUSE_TRIMS = {
909
+ phrases: [],
910
+ directPrefixes: []
911
+ };
912
+ let leadingClauseTrimsCache = null;
913
+ let leadingClauseTrimsPromise = null;
914
+ const loadLeadingClauseTrims = async () => {
915
+ if (leadingClauseTrimsCache) return leadingClauseTrimsCache;
916
+ if (leadingClauseTrimsPromise) return leadingClauseTrimsPromise;
917
+ leadingClauseTrimsPromise = (async () => {
918
+ let data = {};
919
+ try {
920
+ const mod = await import("./legal-form-leading-clauses.mjs");
921
+ data = mod.default ?? mod;
922
+ } catch (err) {
923
+ console.warn("[anonymize] legal-forms: failed to load legal-form-leading-clauses.json:", err);
924
+ }
925
+ const phrases = /* @__PURE__ */ new Set();
926
+ const directPrefixes = /* @__PURE__ */ new Set();
927
+ for (const [key, value] of Object.entries(data)) {
928
+ if (key.startsWith("_") || typeof value !== "object" || value === null) continue;
929
+ const config = value;
930
+ for (const phrase of config.phrases ?? []) if (typeof phrase === "string" && phrase.length > 0) phrases.add(phrase);
931
+ for (const prefix of config.directPrefixes ?? []) if (typeof prefix === "string" && prefix.length > 0) directPrefixes.add(prefix);
932
+ }
933
+ const result = {
934
+ phrases: [...phrases],
935
+ directPrefixes: [...directPrefixes]
936
+ };
937
+ leadingClauseTrimsCache = result;
938
+ return result;
939
+ })();
940
+ return leadingClauseTrimsPromise;
941
+ };
942
+ const getLeadingClauseTrimsSync = () => leadingClauseTrimsCache ?? EMPTY_LEADING_CLAUSE_TRIMS;
893
943
  let legalRoleHeadsCache = null;
894
944
  let legalRoleHeadsPromise = null;
895
945
  const loadLegalRoleHeads = async () => {
@@ -917,7 +967,8 @@ const warmLegalRoleHeads = async () => {
917
967
  loadSentenceVerbIndicators(),
918
968
  loadClauseNounHeads(),
919
969
  loadConnectorProseHeads(),
920
- loadStructuralSingleCapPrefixes()
970
+ loadStructuralSingleCapPrefixes(),
971
+ loadLeadingClauseTrims()
921
972
  ]);
922
973
  };
923
974
  let allLegalSuffixesCache = null;
@@ -1054,13 +1105,12 @@ const loadStructuralSingleCapPrefixes = async () => {
1054
1105
  return structuralSingleCapPrefixesPromise;
1055
1106
  };
1056
1107
  const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
1057
- const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
1108
+ const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, `${HSPACE}+`).replace(/\\\./g, `\\.${HSPACE}?`);
1058
1109
  const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1059
1110
  const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1060
1111
  const buildPatternString = (forms) => {
1061
1112
  if (forms.length === 0) return null;
1062
1113
  const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1063
- const HSPACE = "[^\\S\\n]";
1064
1114
  const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
1065
1115
  const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
1066
1116
  const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
@@ -1068,7 +1118,7 @@ const buildPatternString = (forms) => {
1068
1118
  const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
1069
1119
  const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
1070
1120
  const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
1071
- return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1121
+ return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}${`(?:${HSPACE}+|,${HSPACE}*)`}(?:${alt})(?![${LOWER}])`;
1072
1122
  };
1073
1123
  /**
1074
1124
  * Build legal form regex pattern strings.
@@ -1098,10 +1148,17 @@ const buildLegalFormPatterns = async () => {
1098
1148
  if (longPattern) patterns.push(longPattern);
1099
1149
  const shortPattern = buildPatternString(allForms.filter(isShortForm));
1100
1150
  if (shortPattern) patterns.push(shortPattern);
1101
- const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1102
1151
  const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1103
- patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
1104
- patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
1152
+ const mixedNameSep = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
1153
+ const capitalizedNoDigitPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}`;
1154
+ patterns.push(`${capitalizedNoDigitPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
1155
+ const allcapMixedPrefix = `(?:${ALLCAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD}|\\d{1,4})){1,6}`;
1156
+ patterns.push(`${allcapMixedPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
1157
+ const dottedLineWrapPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}\\.${HSPACE}*\\n${HSPACE}*`;
1158
+ patterns.push(`${dottedLineWrapPrefix}(?:${allcapAlt})(?![${LOWER}])`);
1159
+ const allcapPrefix = `(?:${ALLCAP_WORD})(?:(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}(?:${ALLCAP_WORD})){0,2}`;
1160
+ patterns.push(`${allcapPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
1161
+ patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
1105
1162
  return patterns;
1106
1163
  };
1107
1164
  const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
@@ -1110,13 +1167,14 @@ const UPPER_LETTER_RE = /^\p{Lu}/u;
1110
1167
  const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
1111
1168
  const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
1112
1169
  const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
1113
- const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1114
- const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:[ \\t]+|,[ \\t]*)`, "u");
1115
- const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
1170
+ const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:${HSPACE}+|,${HSPACE}*)`, "u");
1171
+ const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)${HSPACE}+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:${HSPACE}+|,${HSPACE}*)`, "u");
1116
1172
  const isStructuralSingleCapMatch = (text) => {
1117
1173
  const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
1118
1174
  return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
1119
1175
  };
1176
+ const findLastSuffixSeparator = (text) => Math.max(text.lastIndexOf(" "), text.lastIndexOf(" "), text.lastIndexOf("\xA0"), text.lastIndexOf(" "), text.lastIndexOf(","));
1177
+ const stripDocxSpaces = (text) => text.replace(/[  ]/g, "");
1120
1178
  /**
1121
1179
  * Find the word ending just before `pos` in `text`,
1122
1180
  * skipping any whitespace (not newlines).
@@ -1180,6 +1238,58 @@ const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
1180
1238
  entityText: entityText.slice(cut)
1181
1239
  };
1182
1240
  };
1241
+ const splitEmbeddedLegalFormList = (entityStart, entityText) => {
1242
+ const cuts = [0];
1243
+ for (const suffix of getAllLegalSuffixesSync()) {
1244
+ const suffixClean = suffix.replace(/[.,\s]/g, "");
1245
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1246
+ let fromIndex = 0;
1247
+ while (fromIndex < entityText.length) {
1248
+ const suffixStart = entityText.indexOf(suffix, fromIndex);
1249
+ if (suffixStart === -1) break;
1250
+ fromIndex = suffixStart + suffix.length;
1251
+ const suffixEnd = suffixStart + suffix.length;
1252
+ if (suffixEnd >= entityText.length - 1) continue;
1253
+ const afterSuffix = entityText.slice(suffixEnd);
1254
+ const boundary = LEGAL_LIST_BOUNDARY_RE.exec(afterSuffix);
1255
+ if (boundary === null) continue;
1256
+ const nextStart = suffixEnd + boundary[0].length;
1257
+ cuts.push(nextStart);
1258
+ }
1259
+ }
1260
+ const uniqueCuts = [...new Set(cuts)].toSorted((a, b) => a - b);
1261
+ if (uniqueCuts.length === 1) return [{
1262
+ entityStart,
1263
+ entityText
1264
+ }];
1265
+ const segments = [];
1266
+ for (let index = 0; index < uniqueCuts.length; index++) {
1267
+ const start = uniqueCuts[index];
1268
+ const end = uniqueCuts[index + 1] ?? entityText.length;
1269
+ if (start === void 0) continue;
1270
+ const segmentText = entityText.slice(start, end).replace(/[,\s;]+$/u, "");
1271
+ if (segmentText.length === 0) continue;
1272
+ if (!getAllLegalSuffixesSync().some((form) => segmentText.endsWith(form))) continue;
1273
+ segments.push({
1274
+ entityStart: entityStart + start,
1275
+ entityText: segmentText
1276
+ });
1277
+ }
1278
+ return segments;
1279
+ };
1280
+ const hasDisallowedLineBreak = (text) => {
1281
+ for (const match of text.matchAll(/\n/gu)) {
1282
+ const index = match.index;
1283
+ if (index === void 0) continue;
1284
+ const before = text.slice(0, index);
1285
+ const after = text.slice(index + 1);
1286
+ const dottedDesignatorBefore = /\.[^\S\n]*$/u.test(before);
1287
+ const legalSuffixAfter = /^[^\S\n]*(?:\p{Lu}\.[^\S\n]?){1,}\p{Lu}?\.?$/u.test(after);
1288
+ const allCapsSuffixAfter = /^[^\S\n]*\p{Lu}{2,}\.?$/u.test(after);
1289
+ if (!dottedDesignatorBefore || !legalSuffixAfter && !allCapsSuffixAfter) return true;
1290
+ }
1291
+ return false;
1292
+ };
1183
1293
  const hasMiddleInitialBefore = (fullText, pos) => {
1184
1294
  const previousWord = findWordBefore(fullText, pos);
1185
1295
  if (!previousWord) return false;
@@ -1336,7 +1446,37 @@ const extendBackward = (fullText, matchStart, options = {}) => {
1336
1446
  };
1337
1447
  const trimLeadingClause = (text) => {
1338
1448
  let cut = -1;
1339
- for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
1449
+ const trims = getLeadingClauseTrimsSync();
1450
+ const phraseAlternation = trims.phrases.map(escapeForRegex).join("|");
1451
+ if (phraseAlternation.length > 0) {
1452
+ const phraseRe = new RegExp(`(?:^|\\s)(?:${phraseAlternation})${HSPACE}+`, "giu");
1453
+ for (const match of text.matchAll(phraseRe)) cut = Math.max(cut, match.index + match[0].length);
1454
+ }
1455
+ const directPrefixAlternation = trims.directPrefixes.map(escapeForRegex).join("|");
1456
+ const COMMA_GATED_DIRECT_PREFIXES = new Set([
1457
+ "among",
1458
+ "amongst",
1459
+ "between"
1460
+ ]);
1461
+ if (directPrefixAlternation.length > 0) {
1462
+ const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
1463
+ for (const match of text.matchAll(directPrefixRe)) {
1464
+ const matchedPrefix = match[0].trim().toLowerCase();
1465
+ const before = text.slice(0, match.index);
1466
+ if (COMMA_GATED_DIRECT_PREFIXES.has(matchedPrefix) && !/,\s*$/u.test(before)) continue;
1467
+ const words = before.match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
1468
+ if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
1469
+ }
1470
+ }
1471
+ for (const match of text.matchAll(/,/gu)) {
1472
+ const comma = match.index;
1473
+ if (comma === void 0) continue;
1474
+ const before = text.slice(0, comma);
1475
+ if (!/\d/u.test(before)) continue;
1476
+ const after = text.slice(comma + 1);
1477
+ const leadingWs = after.match(/^\s*/u)?.[0].length ?? 0;
1478
+ if ((after.slice(leadingWs).match(/\p{Lu}[\p{L}\p{M}\p{N}]*/gu) ?? []).length >= 3) cut = Math.max(cut, comma + 1 + leadingWs);
1479
+ }
1340
1480
  if (cut <= 0) return {
1341
1481
  offset: 0,
1342
1482
  text
@@ -1419,7 +1559,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1419
1559
  }
1420
1560
  }
1421
1561
  }
1422
- if (processedText.includes("\n")) continue;
1562
+ if (processedText.includes("\n") && hasDisallowedLineBreak(processedText)) continue;
1423
1563
  let entityStart = processedStart;
1424
1564
  let entityText = processedText;
1425
1565
  if (fullText && !trimmed) {
@@ -1429,49 +1569,49 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1429
1569
  entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
1430
1570
  }
1431
1571
  }
1432
- const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
1433
- entityStart = listTrim.entityStart;
1434
- entityText = listTrim.entityText;
1435
- const clauseTrim = trimLeadingClause(entityText);
1436
- if (clauseTrim.offset > 0) {
1437
- entityStart += clauseTrim.offset;
1438
- entityText = clauseTrim.text;
1439
- }
1440
- const getPrefixInfo = (value) => {
1441
- const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1442
- return {
1443
- prefixEnd,
1444
- prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1445
- };
1446
- };
1447
- let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1448
- let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1449
- if (isAllCapsMatch && fullText) {
1450
- const lineStart = fullText.lastIndexOf("\n", entityStart);
1451
- const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1452
- const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1453
- const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1454
- if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1455
- if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1456
- entityStart = match.start;
1457
- entityText = text;
1458
- ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1459
- isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1572
+ for (const segment of splitEmbeddedLegalFormList(entityStart, entityText)) {
1573
+ entityStart = segment.entityStart;
1574
+ entityText = segment.entityText;
1575
+ const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
1576
+ entityStart = listTrim.entityStart;
1577
+ entityText = listTrim.entityText;
1578
+ const clauseTrim = trimLeadingClause(entityText);
1579
+ if (clauseTrim.offset > 0) {
1580
+ entityStart += clauseTrim.offset;
1581
+ entityText = clauseTrim.text;
1460
1582
  }
1461
- } else if (isAllCapsMatch) continue;
1462
- const lastSpace = entityText.lastIndexOf(" ");
1463
- const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
1464
- const suffixClean = rawSuffix.replace(/[.,]/g, "");
1465
- if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1466
- if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
1467
- results.push({
1468
- start: entityStart,
1469
- end: entityStart + entityText.length,
1470
- label: "organization",
1471
- text: entityText,
1472
- score: .95,
1473
- source: DETECTION_SOURCES.LEGAL_FORM
1474
- });
1583
+ if (entityText.includes("\n") && hasDisallowedLineBreak(entityText)) continue;
1584
+ const getPrefixInfo = (value) => {
1585
+ const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1586
+ return {
1587
+ prefixEnd,
1588
+ prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1589
+ };
1590
+ };
1591
+ let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1592
+ let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1593
+ if (isAllCapsMatch && fullText) {
1594
+ if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1595
+ entityStart = match.start;
1596
+ entityText = text;
1597
+ ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1598
+ isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1599
+ }
1600
+ } else if (isAllCapsMatch) continue;
1601
+ const lastSuffixSeparator = findLastSuffixSeparator(entityText);
1602
+ const rawSuffix = lastSuffixSeparator !== -1 ? entityText.slice(lastSuffixSeparator + 1) : "";
1603
+ const suffixClean = rawSuffix.replace(/[.,]/g, "");
1604
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1605
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(stripDocxSpaces(entityText.slice(0, lastSuffixSeparator !== -1 ? lastSuffixSeparator : entityText.length)))) continue;
1606
+ results.push({
1607
+ start: entityStart,
1608
+ end: entityStart + entityText.length,
1609
+ label: "organization",
1610
+ text: entityText,
1611
+ score: .95,
1612
+ source: DETECTION_SOURCES.LEGAL_FORM
1613
+ });
1614
+ }
1475
1615
  }
1476
1616
  return results;
1477
1617
  };
@@ -1885,20 +2025,31 @@ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList
1885
2025
  * with per-language first names and surnames. When
1886
2026
  * omitted, only legacy config files are used.
1887
2027
  */
1888
- const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1889
- if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
2028
+ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
2029
+ const languageKey = languages?.toSorted().join(",") ?? "*";
2030
+ if (ctx.nameCorpus && ctx.nameCorpusKey === languageKey) return Promise.resolve();
2031
+ if (ctx.nameCorpusPromise && ctx.nameCorpusKey === languageKey) return ctx.nameCorpusPromise;
2032
+ ctx.nameCorpus = null;
2033
+ ctx.nameCorpusKey = languageKey;
1890
2034
  const promise = (async () => {
1891
2035
  try {
1892
- const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
2036
+ const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod, commonWordsMod] = await Promise.all([
1893
2037
  import("./names-first.mjs"),
1894
2038
  import("./names-surnames.mjs"),
1895
2039
  import("./names-title-tokens.mjs"),
1896
- import("./names-exclusions.mjs")
2040
+ import("./names-exclusions.mjs"),
2041
+ import("./common-words-en.mjs")
1897
2042
  ]);
1898
2043
  const firstNames = [...legacyFirstMod.default.names];
1899
- if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
2044
+ if (dictionaries?.firstNames) {
2045
+ const entries = languages === void 0 ? Object.entries(dictionaries.firstNames) : Object.entries(dictionaries.firstNames).filter(([language]) => languages.includes(language));
2046
+ for (const [, names] of entries) for (const name of names) firstNames.push(name);
2047
+ }
1900
2048
  const surnames = [...legacySurnameMod.default.names];
1901
- if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
2049
+ if (dictionaries?.surnames) {
2050
+ const entries = languages === void 0 ? Object.entries(dictionaries.surnames) : Object.entries(dictionaries.surnames).filter(([language]) => languages.includes(language));
2051
+ for (const [, names] of entries) for (const name of names) surnames.push(name);
2052
+ }
1902
2053
  const dedup = (arr) => {
1903
2054
  const seen = /* @__PURE__ */ new Set();
1904
2055
  const result = [];
@@ -1909,8 +2060,9 @@ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
1909
2060
  }
1910
2061
  return result;
1911
2062
  };
2063
+ const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
1912
2064
  const dedupFirst = dedup(firstNames);
1913
- const dedupSurnames = dedup(surnames);
2065
+ const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
1914
2066
  const titles = titleMod.default.tokens;
1915
2067
  const exclusions = exclusionMod.default.words;
1916
2068
  ctx.nameCorpus = {
@@ -2007,7 +2159,31 @@ const segmentWords = (fullText) => {
2007
2159
  };
2008
2160
  /** NAME or SURNAME — both represent corpus-matched tokens */
2009
2161
  const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
2010
- const classifyToken = (word, corpus) => {
2162
+ const ALL_CAPS_NAME_LINE_RATIO = .9;
2163
+ const ALL_CAPS_NAME_LINE_MIN_LETTERS = 3;
2164
+ const ALL_CAPS_NAME_LINE_MAX_TOKENS = 6;
2165
+ const isAllCapsLineNameShaped = (fullText, start) => {
2166
+ const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
2167
+ const lineEndIdx = fullText.indexOf("\n", start);
2168
+ const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
2169
+ if (/\d/.test(line)) return false;
2170
+ const tokens = line.match(/\p{L}[\p{L}\p{M}'\-]*/gu) ?? [];
2171
+ return tokens.length > 0 && tokens.length <= ALL_CAPS_NAME_LINE_MAX_TOKENS;
2172
+ };
2173
+ const isAllCapsContextLine = (fullText, start) => {
2174
+ const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
2175
+ const lineEndIdx = fullText.indexOf("\n", start);
2176
+ const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
2177
+ let letters = 0;
2178
+ let upper = 0;
2179
+ for (const ch of line) if (/\p{L}/u.test(ch)) {
2180
+ letters += 1;
2181
+ if (ch === ch.toUpperCase() && ch !== ch.toLowerCase()) upper += 1;
2182
+ }
2183
+ if (letters < ALL_CAPS_NAME_LINE_MIN_LETTERS) return false;
2184
+ return upper / letters >= ALL_CAPS_NAME_LINE_RATIO;
2185
+ };
2186
+ const classifyToken = (word, corpus, fullText) => {
2011
2187
  const { text, start, end } = word;
2012
2188
  const lower = text.toLowerCase();
2013
2189
  const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
@@ -2023,6 +2199,26 @@ const classifyToken = (word, corpus) => {
2023
2199
  start,
2024
2200
  end
2025
2201
  };
2202
+ if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
2203
+ const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
2204
+ const before = fullText.slice(lineStart, start).trimEnd();
2205
+ const lastWord = /\p{L}[\p{L}\p{M}'\-]*$/u.exec(before)?.[0];
2206
+ if (lastWord) {
2207
+ const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus);
2208
+ if (lookup(lastWord)) return {
2209
+ text,
2210
+ type: TOKEN_TYPE.ABBREVIATION,
2211
+ start,
2212
+ end
2213
+ };
2214
+ }
2215
+ return {
2216
+ text,
2217
+ type: TOKEN_TYPE.OTHER,
2218
+ start,
2219
+ end
2220
+ };
2221
+ }
2026
2222
  if (corpus.excludedWords.has(lower)) return {
2027
2223
  text,
2028
2224
  type: TOKEN_TYPE.OTHER,
@@ -2035,12 +2231,29 @@ const classifyToken = (word, corpus) => {
2035
2231
  start,
2036
2232
  end
2037
2233
  };
2038
- if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
2039
- text,
2040
- type: TOKEN_TYPE.OTHER,
2041
- start,
2042
- end
2043
- };
2234
+ if (text.length >= 3 && ALL_UPPER_RE.test(text)) {
2235
+ if (isAllCapsContextLine(fullText, start) && isAllCapsLineNameShaped(fullText, start)) {
2236
+ const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
2237
+ if (isFirstNameToken(titleCased, corpus)) return {
2238
+ text,
2239
+ type: TOKEN_TYPE.NAME,
2240
+ start,
2241
+ end
2242
+ };
2243
+ if (isSurnameToken(titleCased, corpus)) return {
2244
+ text,
2245
+ type: TOKEN_TYPE.SURNAME,
2246
+ start,
2247
+ end
2248
+ };
2249
+ }
2250
+ return {
2251
+ text,
2252
+ type: TOKEN_TYPE.OTHER,
2253
+ start,
2254
+ end
2255
+ };
2256
+ }
2044
2257
  if (!UPPER_START_RE.test(text)) return {
2045
2258
  text,
2046
2259
  type: TOKEN_TYPE.OTHER,
@@ -2085,7 +2298,7 @@ const classifyToken = (word, corpus) => {
2085
2298
  const detectNameCorpus = (fullText, ctx = defaultContext) => {
2086
2299
  const corpus = getCorpus(ctx);
2087
2300
  if (!corpus) return [];
2088
- const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
2301
+ const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus, fullText));
2089
2302
  const entities = [];
2090
2303
  const consumed = /* @__PURE__ */ new Set();
2091
2304
  for (let i = 0; i < tokens.length; i++) {
@@ -2123,6 +2336,8 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
2123
2336
  else if (hasAbbreviation && hasCorpusName) score = .7;
2124
2337
  else if (hasFirstName && chain.length === 1) {
2125
2338
  if (isSentenceStart(fullText, token.start)) continue;
2339
+ const first = chain[0];
2340
+ if (first && ALL_UPPER_RE.test(first.text) && first.text.length >= 3) continue;
2126
2341
  score = .5;
2127
2342
  } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
2128
2343
  else if (hasTitle && chain.length === 1) continue;
@@ -2149,6 +2364,115 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
2149
2364
  return entities;
2150
2365
  };
2151
2366
  //#endregion
2367
+ //#region src/detectors/signatures.ts
2368
+ const SLASH_S_RE = /\/s\/[ \t]*/g;
2369
+ const LABELLED_NAME_RE = /\b(?:By|Name)[ \t]*:[ \t]*(?:\/s\/[ \t]+)?([^\n]*?)(?=\s{3,}|[\t]|\n|$)/gi;
2370
+ const WITNESS_ANCHOR_RE = /\bIN WITNESS WHEREOF\b[^\n]*\n/gi;
2371
+ const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
2372
+ const CAP_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'\\-]{0,30}";
2373
+ const NAME_SHAPE_RE = new RegExp(`^${CAP_TOKEN}(?:[ \\t]+(?:${NAME_PARTICLE}|${CAP_TOKEN})){1,4}$`, "u");
2374
+ const MAX_NAME_LEN = 60;
2375
+ const POST_NOMINAL_SUFFIX_RE = /,\s*(?:Jr|Sr|II|III|IV|V|Esq|Esquire|M\.?D|Ph\.?D|J\.?D|LL\.?M|MBA|CPA|PE|RN|DDS|DVM|DO|MD|CFA|CFP)\.?\s*$/i;
2376
+ const COLUMN_SEPARATOR_RE = /\s{3,}|\t+/;
2377
+ const ORG_SUFFIX_RE = /\b(?:INC\.?|LLC|LLP|LP|CORP\.?|CORPORATION|LTD\.?|GMBH|AG|SE|KG|OHG|SA|SAS|SARL|S\.A\.?|S\.P\.A\.?|PLC|N\.A\.?|N\.V\.?|B\.V\.?|PTY\s+LTD\.?|CO\.|S\.R\.O\.?|A\.S\.?|Z\.S\.?|S\.\s*P\.?|LTDA\.?|EIRELI|EPP|S\/A)\b/i;
2378
+ const IMAGE_STUB_RE = /^(?:\[?img|\[image|\[logo|\(logo\))/i;
2379
+ const normaliseCandidate = (text) => {
2380
+ let candidate = text.trim();
2381
+ candidate = candidate.replace(POST_NOMINAL_SUFFIX_RE, "").trim();
2382
+ return candidate.split(COLUMN_SEPARATOR_RE)[0]?.trim() ?? candidate;
2383
+ };
2384
+ const isNameShape = (text) => {
2385
+ if (text.length === 0 || text.length > MAX_NAME_LEN) return false;
2386
+ if (text.length < 3) return false;
2387
+ if (!NAME_SHAPE_RE.test(text)) return false;
2388
+ return true;
2389
+ };
2390
+ const findLineEnd = (text, pos) => {
2391
+ const idx = text.indexOf("\n", pos);
2392
+ return idx === -1 ? text.length : idx;
2393
+ };
2394
+ const tryEmit = (results, fullText, start, end, score) => {
2395
+ const raw = fullText.slice(start, end);
2396
+ if (ORG_SUFFIX_RE.test(raw)) return false;
2397
+ const candidate = normaliseCandidate(raw);
2398
+ if (!isNameShape(candidate)) return false;
2399
+ const offset = raw.indexOf(candidate);
2400
+ if (offset < 0) return false;
2401
+ const absStart = start + offset;
2402
+ results.push({
2403
+ start: absStart,
2404
+ end: absStart + candidate.length,
2405
+ label: "person",
2406
+ text: candidate,
2407
+ score,
2408
+ source: DETECTION_SOURCES.TRIGGER
2409
+ });
2410
+ return true;
2411
+ };
2412
+ const tryEmitForwardLines = (results, fullText, fromPos, maxLines, score) => {
2413
+ let pos = fromPos;
2414
+ for (let i = 0; i < maxLines; i++) {
2415
+ if (pos >= fullText.length) return false;
2416
+ const lineEnd = findLineEnd(fullText, pos);
2417
+ const line = fullText.slice(pos, lineEnd).trim();
2418
+ if (line.length > 0 && !IMAGE_STUB_RE.test(line)) {
2419
+ if (tryEmit(results, fullText, pos, lineEnd, score)) return true;
2420
+ }
2421
+ pos = lineEnd + 1;
2422
+ }
2423
+ return false;
2424
+ };
2425
+ const findPrevLine = (fullText, pos) => {
2426
+ let cursor = pos - 1;
2427
+ while (cursor >= 0 && fullText.charAt(cursor) !== "\n") cursor--;
2428
+ while (cursor >= 0) {
2429
+ let lineStart = cursor;
2430
+ while (lineStart > 0 && fullText.charAt(lineStart - 1) !== "\n") lineStart -= 1;
2431
+ const lineEnd = cursor;
2432
+ const line = fullText.slice(lineStart, lineEnd).trim();
2433
+ if (line.length > 0 && !IMAGE_STUB_RE.test(line)) return {
2434
+ lineStart,
2435
+ lineEnd
2436
+ };
2437
+ cursor = lineStart - 1;
2438
+ }
2439
+ return null;
2440
+ };
2441
+ const detectSignatures = (fullText, _ctx = defaultContext) => {
2442
+ const results = [];
2443
+ SLASH_S_RE.lastIndex = 0;
2444
+ for (let m = SLASH_S_RE.exec(fullText); m !== null; m = SLASH_S_RE.exec(fullText)) {
2445
+ const afterMark = m.index + m[0].length;
2446
+ const lineEnd = findLineEnd(fullText, afterMark);
2447
+ if (fullText.slice(afterMark, lineEnd).trim().length > 0) {
2448
+ const firstCell = fullText.slice(afterMark, lineEnd).split(COLUMN_SEPARATOR_RE)[0] ?? "";
2449
+ if (firstCell.trim().length > 0) tryEmit(results, fullText, afterMark, afterMark + firstCell.length, .95);
2450
+ } else tryEmitForwardLines(results, fullText, lineEnd + 1, 4, .9);
2451
+ const prev = findPrevLine(fullText, m.index);
2452
+ if (prev) tryEmit(results, fullText, prev.lineStart, prev.lineEnd, .85);
2453
+ }
2454
+ LABELLED_NAME_RE.lastIndex = 0;
2455
+ for (let m = LABELLED_NAME_RE.exec(fullText); m !== null; m = LABELLED_NAME_RE.exec(fullText)) {
2456
+ const value = m[1];
2457
+ if (value === void 0) continue;
2458
+ const valueStart = m.index + m[0].length - value.length;
2459
+ const valueEnd = valueStart + value.length;
2460
+ if (value.trim().length === 0) {
2461
+ tryEmitForwardLines(results, fullText, valueEnd + 1, 3, .9);
2462
+ continue;
2463
+ }
2464
+ tryEmit(results, fullText, valueStart, valueEnd, .95);
2465
+ }
2466
+ WITNESS_ANCHOR_RE.lastIndex = 0;
2467
+ for (let m = WITNESS_ANCHOR_RE.exec(fullText); m !== null; m = WITNESS_ANCHOR_RE.exec(fullText)) {
2468
+ const search = fullText.slice(m.index, m.index + 600);
2469
+ const sentenceEnd = /[.:;]\s*\n|\n\s*\n/.exec(search);
2470
+ if (!sentenceEnd) continue;
2471
+ tryEmitForwardLines(results, fullText, m.index + sentenceEnd.index + sentenceEnd[0].length, 6, .85);
2472
+ }
2473
+ return results;
2474
+ };
2475
+ //#endregion
2152
2476
  //#region src/config/titles.ts
2153
2477
  /**
2154
2478
  * Academic and professional title prefixes.
@@ -2391,9 +2715,14 @@ const STDNUM_ENTRIES = [
2391
2715
  toEntry(at.uid, "tax identification number", .95),
2392
2716
  toEntry(at.tin, "tax identification number", .9),
2393
2717
  toEntry(at.businessid, "registration number", .95),
2718
+ toEntry(ch.uid, "registration number", .95),
2719
+ toEntry(au.abn, "tax identification number", .9),
2720
+ toEntry(au.acn, "registration number", .9),
2394
2721
  toEntry(be.vat, "tax identification number", .95),
2395
2722
  toEntry(be.nn, "national identification number", .9),
2396
2723
  toEntry(nl.vat, "tax identification number", .95),
2724
+ toEntry(no.orgnr, "registration number", .9),
2725
+ toEntry(no.mva, "tax identification number", .95),
2397
2726
  toEntry(dk.vat, "tax identification number", .95),
2398
2727
  toEntry(dk.cpr, "national identification number", .9),
2399
2728
  toEntry(fi.vat, "tax identification number", .95),
@@ -2423,6 +2752,7 @@ const STDNUM_ENTRIES = [
2423
2752
  toEntry(cy.vat, "tax identification number", .95),
2424
2753
  toEntry(mt.vat, "tax identification number", .95),
2425
2754
  toEntry(lu.vat, "tax identification number", .95),
2755
+ toEntry(us.ein, "tax identification number", .9),
2426
2756
  toEntry(br.cpf, "tax identification number", .95),
2427
2757
  toEntry(br.cnpj, "tax identification number", .95)
2428
2758
  ].filter((e) => e !== null);
@@ -2535,6 +2865,30 @@ const ES_CIF = {
2535
2865
  score: .95,
2536
2866
  validator: es.cif
2537
2867
  };
2868
+ const AU_ABN_FORMATTED = {
2869
+ pattern: `\\b\\d{2}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
2870
+ label: "tax identification number",
2871
+ score: .95,
2872
+ validator: au.abn
2873
+ };
2874
+ const NO_ORGNR_FORMATTED = {
2875
+ pattern: `\\b\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
2876
+ label: "registration number",
2877
+ score: .9,
2878
+ validator: no.orgnr
2879
+ };
2880
+ const NO_MVA_FORMATTED = {
2881
+ pattern: "\\bNO[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?MVA\\b",
2882
+ label: "tax identification number",
2883
+ score: .95,
2884
+ validator: no.mva
2885
+ };
2886
+ const US_EIN_FORMATTED = {
2887
+ pattern: `\\b\\d{2}${DASH}\\d{7}\\b`,
2888
+ label: "tax identification number",
2889
+ score: .95,
2890
+ validator: us.ein
2891
+ };
2538
2892
  const BR_CPF_FORMATTED = {
2539
2893
  pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
2540
2894
  label: "tax identification number",
@@ -2655,6 +3009,10 @@ const ALL_REGEX_DEFS = [
2655
3009
  ES_DNI,
2656
3010
  ES_NIE,
2657
3011
  ES_CIF,
3012
+ AU_ABN_FORMATTED,
3013
+ NO_ORGNR_FORMATTED,
3014
+ NO_MVA_FORMATTED,
3015
+ US_EIN_FORMATTED,
2658
3016
  BR_CPF_FORMATTED,
2659
3017
  BR_CNPJ_FORMATTED,
2660
3018
  BR_RG_WITH_SSP,
@@ -3934,6 +4292,47 @@ const MAX_ENTITY_LENGTH = {
3934
4292
  organization: 80,
3935
4293
  person: 60
3936
4294
  };
4295
+ const MAX_ENTITY_WORDS = { organization: 8 };
4296
+ const OPEN_ENDED_SOURCES = new Set(["trigger", "coreference"]);
4297
+ const WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’\-./]*/gu;
4298
+ const countWordTokens = (text) => {
4299
+ let count = 0;
4300
+ WORD_TOKEN_RE.lastIndex = 0;
4301
+ while (WORD_TOKEN_RE.exec(text) !== null) count += 1;
4302
+ return count;
4303
+ };
4304
+ const ALL_CAPS_LINE_LETTER_THRESHOLD = 5;
4305
+ const ALL_CAPS_LINE_RATIO = .95;
4306
+ const ALL_CAPS_LINE_PROSE_EXTRA_LETTERS = 20;
4307
+ const ALL_CAPS_LINE_HEADING_WORD_LIMIT = 5;
4308
+ const SECTION_HEADING_PREFIX_RE = /^\s*(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?\s*\p{Lu}/u;
4309
+ const LINE_LETTER_RE = /\p{L}/gu;
4310
+ const ENTITY_WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’\-]*/gu;
4311
+ const isAllCapsBoilerplateLine = (fullText, start, length) => {
4312
+ const lineStart = fullText.lastIndexOf("\n", start) + 1;
4313
+ const lineEndIdx = fullText.indexOf("\n", start + length);
4314
+ const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
4315
+ let letterCount = 0;
4316
+ let upperCount = 0;
4317
+ let outsideEntityLetters = 0;
4318
+ const entityRelStart = start - lineStart;
4319
+ const entityRelEnd = entityRelStart + length;
4320
+ LINE_LETTER_RE.lastIndex = 0;
4321
+ for (let m = LINE_LETTER_RE.exec(line); m !== null; m = LINE_LETTER_RE.exec(line)) {
4322
+ const ch = m[0];
4323
+ letterCount += 1;
4324
+ if (ch === ch.toUpperCase() && ch !== ch.toLowerCase()) upperCount += 1;
4325
+ if (m.index < entityRelStart || m.index >= entityRelEnd) outsideEntityLetters += 1;
4326
+ }
4327
+ if (letterCount <= ALL_CAPS_LINE_LETTER_THRESHOLD) return false;
4328
+ if (upperCount / letterCount < ALL_CAPS_LINE_RATIO) return false;
4329
+ if (SECTION_HEADING_PREFIX_RE.test(line)) return true;
4330
+ if (outsideEntityLetters >= ALL_CAPS_LINE_PROSE_EXTRA_LETTERS) return true;
4331
+ const entityText = fullText.slice(start, start + length);
4332
+ if ((entityText.match(ENTITY_WORD_TOKEN_RE) ?? []).length > ALL_CAPS_LINE_HEADING_WORD_LIMIT && !entityText.includes(",")) return true;
4333
+ return false;
4334
+ };
4335
+ const isAllCapsCandidate = (text) => text === text.toUpperCase() && /\p{Lu}/u.test(text);
3937
4336
  const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
3938
4337
  const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
3939
4338
  const NUMBER_ABBREV_RE = /(?:^|[\s(])(?:č|čís|nr|no|n)\.\s*$/i;
@@ -4073,6 +4472,9 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
4073
4472
  if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
4074
4473
  const maxLen = MAX_ENTITY_LENGTH[normalized.label];
4075
4474
  if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
4475
+ const maxWords = MAX_ENTITY_WORDS[normalized.label];
4476
+ if (maxWords && OPEN_ENDED_SOURCES.has(normalized.source) && countWordTokens(trimmed) > maxWords) continue;
4477
+ if (fullText && normalized.label === "organization" && isAllCapsCandidate(trimmed) && isAllCapsBoilerplateLine(fullText, normalized.start, trimmed.length)) continue;
4076
4478
  if (SECTION_NUMBER_RE.test(trimmed) && normalized.source !== "trigger") continue;
4077
4479
  if (STANDALONE_YEAR_RE.test(trimmed) && normalized.source !== "trigger") continue;
4078
4480
  if (fullText && normalized.source !== "trigger" && /^\d/.test(trimmed) && NUMBER_ABBREV_RE.test(fullText.slice(Math.max(0, normalized.start - 10), normalized.start))) continue;
@@ -4150,6 +4552,25 @@ const loadAllowList = (ctx) => {
4150
4552
  const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
4151
4553
  /** Sync accessor — returns empty set before init. */
4152
4554
  const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
4555
+ let commonWordsPromise = null;
4556
+ let commonWordsCache = null;
4557
+ const loadCommonWords = () => {
4558
+ if (commonWordsCache) return Promise.resolve(commonWordsCache);
4559
+ if (commonWordsPromise) return commonWordsPromise;
4560
+ commonWordsPromise = (async () => {
4561
+ try {
4562
+ const mod = await import("./common-words-en.mjs");
4563
+ const set = new Set((mod.default?.words ?? []).map((word) => word.toLowerCase()));
4564
+ commonWordsCache = set;
4565
+ return set;
4566
+ } catch {
4567
+ const empty = /* @__PURE__ */ new Set();
4568
+ commonWordsCache = empty;
4569
+ return empty;
4570
+ }
4571
+ })();
4572
+ return commonWordsPromise;
4573
+ };
4153
4574
  /**
4154
4575
  * Curated dictionary entries that are pure dotted
4155
4576
  * single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
@@ -4381,6 +4802,21 @@ const SENTENCE_STARTER_WORDS = new Set([
4381
4802
  const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
4382
4803
  const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
4383
4804
  const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
4805
+ const getCityEntries = (dictionaries, allowedCountries) => {
4806
+ const byCountry = dictionaries?.citiesByCountry;
4807
+ if (!byCountry) return dictionaries?.cities ?? [];
4808
+ const result = [];
4809
+ const append = (entries) => {
4810
+ if (!entries) return;
4811
+ for (const entry of entries) result.push(entry);
4812
+ };
4813
+ if (allowedCountries === null) {
4814
+ for (const entries of Object.values(byCountry)) append(entries);
4815
+ return result;
4816
+ }
4817
+ for (const country of allowedCountries) append(byCountry[country.toUpperCase()]);
4818
+ return result;
4819
+ };
4384
4820
  /**
4385
4821
  * Resolve which dictionaries to load based on country
4386
4822
  * and category filters, then build the deny list data.
@@ -4392,21 +4828,24 @@ const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^
4392
4828
  * Returns null if no dictionaries are provided.
4393
4829
  */
4394
4830
  const buildDenyList = async (config, ctx = defaultContext) => {
4395
- await initNameCorpus(ctx, config.dictionaries);
4831
+ await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
4396
4832
  await Promise.all([
4397
4833
  loadStopwords(ctx),
4398
4834
  loadAllowList(ctx),
4399
4835
  loadPersonStopwords(ctx),
4400
4836
  loadAddressStopwords(ctx),
4837
+ loadCommonWords(),
4401
4838
  loadStreetTypeRe(),
4402
4839
  loadGenericRoles(ctx)
4403
4840
  ]);
4841
+ const commonWords = await loadCommonWords();
4404
4842
  const dictionaries = config.dictionaries;
4405
4843
  const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
4406
- const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
4407
4844
  const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
4408
- if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
4409
4845
  const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
4846
+ const cityEntries = getCityEntries(dictionaries, allowedCountries);
4847
+ const hasCities = cityEntries.length > 0;
4848
+ if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
4410
4849
  const excluded = config.denyListExcludeCategories;
4411
4850
  const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
4412
4851
  const patternList = [];
@@ -4417,8 +4856,11 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4417
4856
  const addDenyListEntry = (entry, label, source = "deny-list") => {
4418
4857
  const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
4419
4858
  if (normalized.length === 0) return;
4420
- if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
4421
4859
  const lower = normalized.toLowerCase();
4860
+ if (source !== "custom-deny-list" && label !== "address") {
4861
+ if (SINGLE_WORD_RE.test(normalized) && commonWords.has(lower)) return;
4862
+ if (isShortCuratedNoiseAcronym(normalized)) return;
4863
+ }
4422
4864
  const existing = patternIndex.get(lower);
4423
4865
  if (existing !== void 0) {
4424
4866
  if (!labelList[existing].includes(label)) labelList[existing].push(label);
@@ -4435,10 +4877,12 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4435
4877
  if (hasDenyList) {
4436
4878
  const denyListData = dictionaries.denyList;
4437
4879
  const metaData = dictionaries.denyListMeta;
4880
+ const useScopedNameCorpus = config.nameCorpusLanguages !== void 0;
4438
4881
  for (const [id, entries] of Object.entries(denyListData)) {
4439
4882
  const meta = metaData[id];
4440
4883
  if (!meta) continue;
4441
4884
  if (!config.enableNameCorpus && meta.category === "Names") continue;
4885
+ if (useScopedNameCorpus && meta.category === "Names") continue;
4442
4886
  if (excludeCategories.has(meta.category)) continue;
4443
4887
  if (allowedCountries !== null && meta.country !== null) {
4444
4888
  if (!allowedCountries.has(meta.country)) continue;
@@ -4446,7 +4890,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4446
4890
  for (const entry of entries) addDenyListEntry(entry, meta.label);
4447
4891
  }
4448
4892
  }
4449
- if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
4893
+ if (hasCities && !excludeCategories.has("Places")) for (const entry of cityEntries) addDenyListEntry(entry, "address", "city");
4450
4894
  if (hasCustomDenyList) for (const entry of config.customDenyList) {
4451
4895
  addDenyListEntry(entry.value, entry.label, "custom-deny-list");
4452
4896
  for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
@@ -4541,8 +4985,8 @@ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
4541
4985
  * the search instance was built on a different context
4542
4986
  * (e.g. cachedSearch).
4543
4987
  */
4544
- const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
4545
- await initNameCorpus(ctx, dictionaries);
4988
+ const ensureDenyListData = async (ctx = defaultContext, dictionaries, nameCorpusLanguages) => {
4989
+ await initNameCorpus(ctx, dictionaries, nameCorpusLanguages);
4546
4990
  await Promise.all([
4547
4991
  loadStopwords(ctx),
4548
4992
  loadAllowList(ctx),
@@ -4721,7 +5165,7 @@ const extendCityDistricts = (entities, fullText) => {
4721
5165
  entity.text = fullText.slice(entity.start, entity.end);
4722
5166
  }
4723
5167
  const afterDistrict = fullText.slice(entity.end);
4724
- const dashDistrictM = /^[\s]*[-–][\s]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
5168
+ const dashDistrictM = /^[ \t]{1,4}[-–][ \t]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
4725
5169
  if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
4726
5170
  entity.end += dashDistrictM[0].length;
4727
5171
  entity.text = fullText.slice(entity.start, entity.end);
@@ -4962,6 +5406,7 @@ const hasHouseNumberNearStreetWord = (fullText, seed) => {
4962
5406
  const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
4963
5407
  return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
4964
5408
  };
5409
+ const isLowercaseStreetWordInProse = (fullText, seed) => /^\p{Ll}/u.test(seed.text) && /^\s+\p{Ll}/u.test(fullText.slice(seed.end, seed.end + 16)) && !hasHouseNumberNearStreetWord(fullText, seed);
4965
5410
  const getUsZipPlusFourContext = (fullText, start, seeds) => {
4966
5411
  const stateSeed = getUsStateSeedBeforeZip(fullText, start);
4967
5412
  if (stateSeed !== null) return {
@@ -5022,12 +5467,14 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
5022
5467
  for (const match of allMatches) {
5023
5468
  const idx = match.pattern;
5024
5469
  if (idx < sliceStart || idx >= sliceEnd) continue;
5025
- seeds.push({
5470
+ const seed = {
5026
5471
  type: "street-word",
5027
5472
  start: match.start,
5028
5473
  end: match.end,
5029
5474
  text: match.text
5030
- });
5475
+ };
5476
+ if (isLowercaseStreetWordInProse(fullText, seed)) continue;
5477
+ seeds.push(seed);
5031
5478
  }
5032
5479
  for (const e of existingEntities) {
5033
5480
  if (e.label !== "address") continue;
@@ -5166,6 +5613,7 @@ const NON_ADDRESS_LABELS = new Set([
5166
5613
  ]);
5167
5614
  const expandCluster = async (fullText, cluster, existingEntities) => {
5168
5615
  const { start, end } = cluster;
5616
+ const seedTypes = new Set(cluster.seeds.map((seed) => seed.type));
5169
5617
  let leftBound = 0;
5170
5618
  for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
5171
5619
  let leftPos = start;
@@ -5180,6 +5628,10 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
5180
5628
  if (fullText.slice(p + 1, leftPos).includes("\n")) break;
5181
5629
  leftPos = p + 1;
5182
5630
  }
5631
+ if (!(seedTypes.has("street-word") || seedTypes.has("house-number") || seedTypes.has("postal-code") || seedTypes.has("address-trigger"))) return {
5632
+ start: Math.min(leftPos, start),
5633
+ end
5634
+ };
5183
5635
  let rightPos = end;
5184
5636
  const remaining = fullText.slice(rightPos);
5185
5637
  let nearestBoundary = Math.min(remaining.length, 200);
@@ -6088,7 +6540,7 @@ const mergeAdjacent = (entities, fullText) => {
6088
6540
  break;
6089
6541
  }
6090
6542
  }
6091
- if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev) && isLegalFormOrganization(entity) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
6543
+ if (!hasLockedBoundary$1(prev) && !((isLegalFormOrganization(prev) || isLegalFormOrganization(entity)) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
6092
6544
  prev.end = entity.end;
6093
6545
  prev.text = fullText.slice(prev.start, prev.end);
6094
6546
  prev.score = Math.max(prev.score, entity.score);
@@ -6246,9 +6698,12 @@ const enforceBoundaryConsistency = (entities, fullText) => {
6246
6698
  //#region src/build-unified-search.ts
6247
6699
  const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
6248
6700
  const ALNUM_RE = /[\p{L}\p{N}]/u;
6701
+ const createAllowedLabelSet$1 = (labels) => labels.length > 0 ? new Set(labels) : null;
6702
+ const labelIsAllowed$1 = (label, allowedLabels) => allowedLabels === null || allowedLabels.has(label);
6249
6703
  const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
6250
6704
  const legalFormsEnabled = isLegalFormsEnabled(config);
6251
- const customRegexes = config.enableRegex ? config.customRegexes ?? [] : [];
6705
+ const allowedLabels = createAllowedLabelSet$1(config.enableHotwordRules === true ? expandLabelsForHotwordRules(config.labels) : config.labels);
6706
+ const customRegexes = config.enableRegex ? (config.customRegexes ?? []).filter((entry) => labelIsAllowed$1(entry.label, allowedLabels)) : [];
6252
6707
  const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
6253
6708
  legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
6254
6709
  config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
@@ -6257,22 +6712,30 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
6257
6712
  }),
6258
6713
  config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
6259
6714
  buildStreetTypePatterns(),
6260
- getCurrencyPatterns(),
6261
- getDatePatterns(),
6262
- getSigningClausePatterns()
6715
+ config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ? getCurrencyPatterns() : Promise.resolve([]),
6716
+ config.enableRegex && labelIsAllowed$1("date", allowedLabels) ? getDatePatterns() : Promise.resolve([]),
6717
+ config.enableRegex && labelIsAllowed$1("address", allowedLabels) ? getSigningClausePatterns() : Promise.resolve([])
6263
6718
  ]);
6264
- const allRegex = [
6265
- ...REGEX_PATTERNS,
6266
- ...currencyPatterns,
6267
- ...datePatterns,
6268
- ...signingPatterns
6269
- ];
6270
- const regexMeta = [
6271
- ...REGEX_META,
6272
- ...currencyPatterns.map(() => CURRENCY_PATTERN_META),
6273
- ...datePatterns.map(() => DATE_PATTERN_META),
6274
- ...signingPatterns.map(() => SIGNING_CLAUSE_META)
6275
- ];
6719
+ const allRegex = [];
6720
+ const regexMeta = [];
6721
+ if (config.enableRegex) for (const [index, pattern] of REGEX_PATTERNS.entries()) {
6722
+ const meta = REGEX_META[index];
6723
+ if (!meta || !labelIsAllowed$1(meta.label, allowedLabels)) continue;
6724
+ allRegex.push(pattern);
6725
+ regexMeta.push(meta);
6726
+ }
6727
+ for (const pattern of currencyPatterns) {
6728
+ allRegex.push(pattern);
6729
+ regexMeta.push(CURRENCY_PATTERN_META);
6730
+ }
6731
+ for (const pattern of datePatterns) {
6732
+ allRegex.push(pattern);
6733
+ regexMeta.push(DATE_PATTERN_META);
6734
+ }
6735
+ for (const pattern of signingPatterns) {
6736
+ allRegex.push(pattern);
6737
+ regexMeta.push(SIGNING_CLAUSE_META);
6738
+ }
6276
6739
  const customRegexMeta = customRegexes.map((entry) => ({
6277
6740
  label: entry.label,
6278
6741
  score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
@@ -6764,7 +7227,7 @@ const configKey = (config, gazetteerEntries) => {
6764
7227
  score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
6765
7228
  })).sort().join("\n") : "";
6766
7229
  const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
6767
- return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
7230
+ return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.nameCorpusLanguages?.toSorted().join(",") ?? ""}:${config.enableRegex}:${config.labels.toSorted().join(",")}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
6768
7231
  };
6769
7232
  /**
6770
7233
  * Get or build a cached search instance. Cache state
@@ -6784,6 +7247,13 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
6784
7247
  return result;
6785
7248
  };
6786
7249
  /**
7250
+ * Pre-build and cache the unified search instance for a
7251
+ * pipeline configuration. Use the same context in
7252
+ * `runPipeline` to reuse the prepared automata without
7253
+ * passing `cachedSearch` around manually.
7254
+ */
7255
+ const preparePipelineSearch = ({ config, gazetteerEntries = [], context }) => getCachedSearch(config, gazetteerEntries, context ?? defaultContext);
7256
+ /**
6787
7257
  * Run the full detection pipeline.
6788
7258
  *
6789
7259
  * Two TextSearch instances scan the text (regex +
@@ -6840,7 +7310,7 @@ const runPipeline = async (options) => {
6840
7310
  warmAddressStopKeywords(),
6841
7311
  hotwordInit
6842
7312
  ]);
6843
- if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
7313
+ if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries, config.nameCorpusLanguages);
6844
7314
  let zones = [];
6845
7315
  if (config.enableZoneClassification && zoneInitOk) {
6846
7316
  zones = classifyZones(fullText, ctx);
@@ -6849,7 +7319,11 @@ const runPipeline = async (options) => {
6849
7319
  checkAbort(signal);
6850
7320
  const hotwordsActive = enableHotwords && hotwordInitOk;
6851
7321
  const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
6852
- const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
7322
+ const searchConfig = hotwordsActive ? {
7323
+ ...config,
7324
+ labels: [...expandLabelsForHotwordRules(config.labels)]
7325
+ } : config;
7326
+ const search = cachedSearch ?? await getCachedSearch(searchConfig, gazetteerEntries, ctx);
6853
7327
  checkAbort(signal);
6854
7328
  const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
6855
7329
  const { slices } = search;
@@ -6865,11 +7339,13 @@ const runPipeline = async (options) => {
6865
7339
  const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
6866
7340
  const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
6867
7341
  if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
7342
+ const signatureEntities = filterAllowedLabels(detectSignatures(fullText, ctx), preHotwordAllowedLabels);
7343
+ if (signatureEntities.length > 0) log("signatures", `${signatureEntities.length} matches`);
6868
7344
  checkAbort(signal);
6869
7345
  let rawNameCorpusEntities = [];
6870
7346
  let nameCorpusEntities = [];
6871
7347
  if (config.enableNameCorpus && !config.enableDenyList) {
6872
- await initNameCorpus(ctx, config.dictionaries);
7348
+ await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
6873
7349
  checkAbort(signal);
6874
7350
  rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
6875
7351
  nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
@@ -6921,6 +7397,7 @@ const runPipeline = async (options) => {
6921
7397
  checkAbort(signal);
6922
7398
  const preAddressEntities = [
6923
7399
  ...triggerEntities,
7400
+ ...signatureEntities,
6924
7401
  ...regexEntities,
6925
7402
  ...legalFormEntities,
6926
7403
  ...nameCorpusEntities,
@@ -7568,6 +8045,6 @@ const levenshtein = (rawA, rawB) => {
7568
8045
  //#region src/wasm.ts
7569
8046
  initTextSearch(TextSearch);
7570
8047
  //#endregion
7571
- export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
8048
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
7572
8049
 
7573
8050
  //# sourceMappingURL=wasm.mjs.map