@stll/anonymize 1.0.3 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -73,6 +73,7 @@ const DEFAULT_ENTITY_LABELS = [
73
73
  "monetary amount",
74
74
  "land parcel"
75
75
  ];
76
+ const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
76
77
  //#endregion
77
78
  //#region src/context.ts
78
79
  /**
@@ -293,6 +294,33 @@ const getDefinitionPatterns = async (ctx) => {
293
294
  };
294
295
  const SEARCH_WINDOW = 200;
295
296
  /**
297
+ * Check whether an alias has textual similarity to
298
+ * the source entity. Prevents roles and structural
299
+ * terms from being treated as name aliases.
300
+ *
301
+ * Three checks (any passes → similar):
302
+ * 1. Word overlap: a word in the alias appears in the
303
+ * entity (case-insensitive, min 2 chars)
304
+ * 2. Initials: alias letters match first letters of
305
+ * entity words ("TB" ↔ "Tomas Bata")
306
+ * 3. Substring: alias is a substring of the entity
307
+ * or vice versa (min 3 chars)
308
+ */
309
+ const hasEntitySimilarity = (alias, entityText) => {
310
+ const aliasLower = alias.toLowerCase();
311
+ const entityLower = entityText.toLowerCase();
312
+ if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
313
+ if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
314
+ const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
315
+ const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
316
+ const entityWordSet = new Set(entityWords);
317
+ for (const word of aliasWords) if (entityWordSet.has(word)) return true;
318
+ if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
319
+ for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
320
+ }
321
+ return false;
322
+ };
323
+ /**
296
324
  * Scan for defined-term patterns near known entities.
297
325
  *
298
326
  * Legal documents universally follow:
@@ -332,6 +360,9 @@ const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) =>
332
360
  break;
333
361
  }
334
362
  if (bestEntity === null) continue;
363
+ const gapText = fullText.slice(bestEntity.end, defPos);
364
+ if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
365
+ if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
335
366
  const key = `${alias.toLowerCase()}::${bestEntity.label}`;
336
367
  if (seen.has(key)) continue;
337
368
  seen.add(key);
@@ -697,6 +728,8 @@ const TOKEN_TYPE = {
697
728
  CAPITALIZED: "capitalized",
698
729
  OTHER: "other"
699
730
  };
731
+ const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
732
+ const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
700
733
  /**
701
734
  * Check if a token is in the first-name set, either
702
735
  * directly or after stripping Czech/Slovak inflection.
@@ -828,7 +861,9 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
828
861
  if (!next) break;
829
862
  const prev = chain.at(-1);
830
863
  if (prev) {
831
- if (fullText.slice(prev.end, next.start).includes("\n")) break;
864
+ const gap = fullText.slice(prev.end, next.start);
865
+ const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
866
+ if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
832
867
  }
833
868
  if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
834
869
  chain.push(next);
@@ -1396,6 +1431,7 @@ const buildDatePatternsFromMonths = (alt) => {
1396
1431
  return [
1397
1432
  `(?i)\\b\\d{1,2}\\.?\\s+(?:${alt})\\.?\\s+\\d{4}(?:\\s+\\d{1,2}:\\d{2}(?::\\d{2})?)?\\b`,
1398
1433
  `(?i)\\b(?:${alt})\\.?\\s+\\d{1,2},?\\s+\\d{4}\\b`,
1434
+ `(?i)\\b(?:${alt})\\.?\\s+\\d{1,2}(?=\\s|[.,;!?)]|$)`,
1399
1435
  `(?i)\\b\\d{1,2}(?:st|nd|rd|th)\\s+(?:${alt})\\.?(?:\\s+\\d{4})?(?=\\s|[.,;!?)]|$)`,
1400
1436
  `(?i)\\b(?:${alt})\\.?\\s+\\d{4}\\b`,
1401
1437
  `(?i)\\b\\d{4}\\.\\s+(?:${alt})\\.?\\s+\\d{1,2}\\.?(?=\\s|[.,;!?)]|$)`,
@@ -1462,9 +1498,11 @@ const buildCurrencyPatterns = (data) => {
1462
1498
  if (!symbols && !trailingAlt) return [];
1463
1499
  const NUM = "(?:\\d{1,3}(?:[,.'[^\\S\\n\\t]]\\d{3})+|\\d{1,9})";
1464
1500
  const patterns = [];
1465
- if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}(?:[.,][^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2})?)?\\b`);
1466
- if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}(?:[.,][^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2})?)?\\b`);
1467
- if (trailingAlt) patterns.push(`\\b${NUM}(?:[.,][^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2})?)?[^\\S\\n\\t]{0,4}(?:${trailingAlt})(?:\\b|(?=\\s|[.,;!?)]|$))`);
1501
+ const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
1502
+ const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
1503
+ if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${END}`);
1504
+ if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${END}`);
1505
+ if (trailingAlt) patterns.push(`\\b${NUM}${DECIMAL}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${END}`);
1468
1506
  return patterns;
1469
1507
  };
1470
1508
  /** Cached promise for currency patterns. Loaded once. */
@@ -1572,10 +1610,10 @@ const ALLCAP_WORD = `[${UPPER}]{2,}`;
1572
1610
  const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
1573
1611
  const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
1574
1612
  const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1575
- const buildPatternString = (forms, requireCapBefore) => {
1613
+ const buildPatternString = (forms) => {
1576
1614
  if (forms.length === 0) return null;
1577
1615
  const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1578
- return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|\\s+(?:a|and|und|et|e|y|i)\\s+)`}(?:${ANY_WORD})){0,10}`}${requireCapBefore ? `(?:\\s+|,\\s*)` : `\\s+`}(?:${alt})(?![${LOWER}])`;
1616
+ return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1579
1617
  };
1580
1618
  /**
1581
1619
  * Build legal form regex pattern strings.
@@ -1600,15 +1638,82 @@ const buildLegalFormPatterns = async () => {
1600
1638
  }
1601
1639
  }
1602
1640
  const patterns = [];
1603
- const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)), false);
1641
+ const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
1604
1642
  if (longPattern) patterns.push(longPattern);
1605
- const shortPattern = buildPatternString(allForms.filter(isShortForm), true);
1643
+ const shortPattern = buildPatternString(allForms.filter(isShortForm));
1606
1644
  if (shortPattern) patterns.push(shortPattern);
1607
1645
  const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1608
1646
  const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1609
1647
  patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
1610
1648
  return patterns;
1611
1649
  };
1650
+ const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
1651
+ const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1652
+ /**
1653
+ * Find the word ending just before `pos` in `text`,
1654
+ * skipping any whitespace (not newlines).
1655
+ * Returns null if no word is found (e.g., at start
1656
+ * of text, or preceded by non-word chars like ".").
1657
+ */
1658
+ const findWordBefore = (text, pos) => {
1659
+ let scan = pos - 1;
1660
+ while (scan >= 0) {
1661
+ const ch = text.charAt(scan);
1662
+ if (ch === "\n" || !/\s/.test(ch)) break;
1663
+ scan--;
1664
+ }
1665
+ if (scan < 0 || text.charAt(scan) === "\n") return null;
1666
+ const wordEnd = scan + 1;
1667
+ while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
1668
+ const wordStart = scan + 1;
1669
+ const word = text.slice(wordStart, wordEnd);
1670
+ if (word.length === 0) return null;
1671
+ return {
1672
+ word,
1673
+ start: wordStart
1674
+ };
1675
+ };
1676
+ /**
1677
+ * Extend a match backward through uppercase words and
1678
+ * lowercase connectors. Stops at start of text,
1679
+ * newline, or a word that doesn't qualify.
1680
+ *
1681
+ * Connectors (a, and, und, et, ...) are only consumed
1682
+ * when there is a valid word before them — a trailing
1683
+ * connector at an entity boundary is not consumed.
1684
+ */
1685
+ const extendBackward = (fullText, matchStart) => {
1686
+ let pos = matchStart;
1687
+ while (pos > 0) {
1688
+ const found = findWordBefore(fullText, pos);
1689
+ if (!found) break;
1690
+ const { word, start: wordStart } = found;
1691
+ const isUpper = /^\p{Lu}/u.test(word);
1692
+ const isConnector = CONNECTOR_RE.test(word);
1693
+ if (isUpper) pos = wordStart;
1694
+ else if (isConnector) {
1695
+ const prev = findWordBefore(fullText, wordStart);
1696
+ if (!prev) break;
1697
+ if (!/^\p{Lu}/u.test(prev.word)) break;
1698
+ pos = prev.start;
1699
+ } else break;
1700
+ }
1701
+ return pos;
1702
+ };
1703
+ const trimLeadingClause = (text) => {
1704
+ let cut = -1;
1705
+ for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
1706
+ if (cut <= 0) return {
1707
+ offset: 0,
1708
+ text
1709
+ };
1710
+ const trimmed = text.slice(cut);
1711
+ const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
1712
+ return {
1713
+ offset: cut + leadingWs,
1714
+ text: trimmed.slice(leadingWs)
1715
+ };
1716
+ };
1612
1717
  /**
1613
1718
  * Process legal form matches from the unified search.
1614
1719
  * Receives all matches; filters to the legal forms
@@ -1622,27 +1727,52 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1622
1727
  const text = match.text.trimEnd();
1623
1728
  if (text.length < 5) continue;
1624
1729
  if (text.includes("\n")) continue;
1625
- const prefixEnd = text.lastIndexOf(",") !== -1 ? text.lastIndexOf(",") : text.lastIndexOf(" ");
1626
- const prefixPart = prefixEnd > 0 ? text.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : text.replace(/[^a-zA-ZÀ-ž]/g, "");
1627
- const isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1730
+ let entityStart = match.start;
1731
+ let entityText = text;
1732
+ if (fullText) {
1733
+ const extended = extendBackward(fullText, match.start);
1734
+ if (extended < match.start) {
1735
+ entityStart = extended;
1736
+ entityText = fullText.slice(extended, match.start + text.length).trimEnd();
1737
+ }
1738
+ }
1739
+ const clauseTrim = trimLeadingClause(entityText);
1740
+ if (clauseTrim.offset > 0) {
1741
+ entityStart += clauseTrim.offset;
1742
+ entityText = clauseTrim.text;
1743
+ }
1744
+ const getPrefixInfo = (value) => {
1745
+ const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
1746
+ return {
1747
+ prefixEnd,
1748
+ prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
1749
+ };
1750
+ };
1751
+ let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
1752
+ let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1628
1753
  if (isAllCapsMatch && fullText) {
1629
- const lineStart = fullText.lastIndexOf("\n", match.start);
1630
- const lineEnd = fullText.indexOf("\n", match.end);
1754
+ const lineStart = fullText.lastIndexOf("\n", entityStart);
1755
+ const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
1631
1756
  const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
1632
1757
  const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
1633
1758
  if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
1634
- if ((prefixPart.length > 0 ? text.slice(0, prefixEnd > 0 ? prefixEnd : text.length).trim().split(/\s+/).length : 0) > 3) continue;
1759
+ if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
1760
+ entityStart = match.start;
1761
+ entityText = text;
1762
+ ({prefixEnd, prefixPart} = getPrefixInfo(entityText));
1763
+ isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
1764
+ }
1635
1765
  } else if (isAllCapsMatch) continue;
1636
- const lastSpace = text.lastIndexOf(" ");
1637
- const rawSuffix = lastSpace !== -1 ? text.slice(lastSpace + 1) : "";
1766
+ const lastSpace = entityText.lastIndexOf(" ");
1767
+ const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
1638
1768
  const suffixClean = rawSuffix.replace(/[.,]/g, "");
1639
1769
  if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1640
- if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(text.slice(0, lastSpace !== -1 ? lastSpace : text.length))) continue;
1770
+ if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
1641
1771
  results.push({
1642
- start: match.start,
1643
- end: match.start + text.length,
1772
+ start: entityStart,
1773
+ end: entityStart + entityText.length,
1644
1774
  label: "organization",
1645
- text,
1775
+ text: entityText,
1646
1776
  score: .95,
1647
1777
  source: DETECTION_SOURCES.LEGAL_FORM
1648
1778
  });
@@ -1818,6 +1948,33 @@ const buildTriggerPatterns = async () => {
1818
1948
  } catch (err) {
1819
1949
  if (!(err instanceof Error) || !err.message.includes("Cannot find module")) throw err;
1820
1950
  }
1951
+ try {
1952
+ const yearMod = await import("@stll/anonymize-data/config/year-words.json");
1953
+ const data = yearMod.default ?? yearMod;
1954
+ const seen = /* @__PURE__ */ new Set();
1955
+ const yearValidation = compileValidations([{
1956
+ type: "matches-pattern",
1957
+ pattern: "^(?:19|20)\\d{2}\\.?$"
1958
+ }]);
1959
+ for (const [key, words] of Object.entries(data)) {
1960
+ if (key.startsWith("_") || !Array.isArray(words)) continue;
1961
+ for (const word of words) {
1962
+ const lc = word.toLowerCase();
1963
+ if (seen.has(lc)) continue;
1964
+ seen.add(lc);
1965
+ rules.push({
1966
+ trigger: word,
1967
+ label: "date",
1968
+ strategy: {
1969
+ type: "n-words",
1970
+ count: 1
1971
+ },
1972
+ validations: yearValidation,
1973
+ includeTrigger: false
1974
+ });
1975
+ }
1976
+ }
1977
+ } catch {}
1821
1978
  const seen = /* @__PURE__ */ new Map();
1822
1979
  for (const rule of rules) {
1823
1980
  const key = rule.trigger.toLowerCase();
@@ -1851,7 +2008,11 @@ const stripQuotes = (value) => {
1851
2008
  };
1852
2009
  };
1853
2010
  /** Hard stop characters for to-next-comma scanning. */
1854
- const COMMA_STOP_CHARS = new Set(["\n", "("]);
2011
+ const COMMA_STOP_CHARS = new Set([
2012
+ "\n",
2013
+ "(",
2014
+ " "
2015
+ ]);
1855
2016
  /**
1856
2017
  * Field-label keywords that terminate address scanning.
1857
2018
  * When a comma in the address strategy is followed by
@@ -2344,6 +2505,56 @@ const MAX_ENTITY_LENGTH = {
2344
2505
  };
2345
2506
  const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
2346
2507
  const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
2508
+ const NUMBER_ABBREV_RE = /(?:^|[\s(])(?:č|čís|nr|no|n)\.\s*$/i;
2509
+ const SIGNING_CLAUSE_ADDRESS_RE = /^(?:v|ve)\s+[^\d,\n]{1,40},?\s+dne$/iu;
2510
+ const PERSON_TRAILING_NOUNS = new Set([
2511
+ "association",
2512
+ "period",
2513
+ "reform"
2514
+ ]);
2515
+ const LEGAL_FORM_HEADING_RE = /\b(?:agreement|amendment|contract|exhibit)\b/iu;
2516
+ const LEADING_ARTIFACT_RE = /^(?:\.\s)+/u;
2517
+ const ADDRESS_ROLE_PREFIX_RE = /^(?:prodávajícího|kupujícího|objednatele|zhotovitele|pronajímatele|dodavatele|odběratele|zaměstnance|zaměstnavatele|nájemce)\s+(?=(?:\p{Lu}|\d|ul\.?|ulice|nám\.?|náměstí|tř\.?|třída|nábř\.?|nábřeží|č\.p\.?|č\.ev\.?|sídliště))/iu;
2518
+ const ADDRESS_INLINE_ABBREV_AFTER_RE = /^(?:\p{Lu}[\p{L}\p{M}]{0,3}\.|ul\.?|nám\.?|tř\.?|nábř\.?|č\.p\.?|č\.ev\.?)/u;
2519
+ const ADDRESS_INLINE_ABBREV_BEFORE_RE = /(?:^|[\s,])(?:st|ave|rd|dr|blvd|ln|hwy|pkwy|cir|ct|pl|sq|ter|trl|ste|apt|bldg|fl|ul|nám|tř|nábř|č\.p|č\.ev)$/iu;
2520
+ const ADDRESS_CONTINUATION_WORD_RE = /^(?:suite|building|floor|unit|apartment|room|tower|wing|block|bldg|ste|apt|fl)\b/iu;
2521
+ const trimTrailingAddressProse = (text) => {
2522
+ for (const match of text.matchAll(/\.(?=\s+\p{Lu})/gu)) {
2523
+ const cutoff = match.index;
2524
+ if (cutoff === void 0) continue;
2525
+ const before = text.slice(0, cutoff);
2526
+ if (!HAS_DIGIT_RE.test(before)) continue;
2527
+ const after = text.slice(cutoff + 1).trimStart();
2528
+ if (after.length < 5 || ADDRESS_INLINE_ABBREV_AFTER_RE.test(after) || ADDRESS_INLINE_ABBREV_BEFORE_RE.test(before.trimEnd()) || ADDRESS_CONTINUATION_WORD_RE.test(after)) continue;
2529
+ return before.trimEnd();
2530
+ }
2531
+ return text;
2532
+ };
2533
+ const normalizeEntity = (entity) => {
2534
+ let start = entity.start;
2535
+ let text = entity.text;
2536
+ const trimLeading = (re) => {
2537
+ const match = re.exec(text);
2538
+ if (!match) return;
2539
+ start += match[0].length;
2540
+ text = text.slice(match[0].length);
2541
+ };
2542
+ trimLeading(LEADING_ARTIFACT_RE);
2543
+ trimLeading(/^\s+/u);
2544
+ if (entity.label === "address") {
2545
+ trimLeading(ADDRESS_ROLE_PREFIX_RE);
2546
+ text = trimTrailingAddressProse(text);
2547
+ }
2548
+ const trailingMatch = /[,\s]+$/u.exec(text);
2549
+ if (trailingMatch) text = text.slice(0, text.length - trailingMatch[0].length);
2550
+ if (text.length === 0) return null;
2551
+ return {
2552
+ ...entity,
2553
+ start,
2554
+ end: start + text.length,
2555
+ text
2556
+ };
2557
+ };
2347
2558
  const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
2348
2559
  /**
2349
2560
  * Load generic-roles.json and cache the result on the
@@ -2377,21 +2588,32 @@ const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
2377
2588
  * Runs as a post-processing step after all detection
2378
2589
  * layers have merged.
2379
2590
  */
2380
- const filterFalsePositives = (entities, ctx = defaultContext) => {
2591
+ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
2381
2592
  const filtered = [];
2382
2593
  const roles = getGenericRoles(ctx);
2383
2594
  for (const entity of entities) {
2384
- const trimmed = entity.text.trim();
2595
+ const normalized = normalizeEntity(entity);
2596
+ if (!normalized) continue;
2597
+ const trimmed = normalized.text;
2385
2598
  if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
2386
- const maxLen = MAX_ENTITY_LENGTH[entity.label];
2387
- if (maxLen && trimmed.length > maxLen && entity.source !== "legal-form") continue;
2388
- if (SECTION_NUMBER_RE.test(trimmed) && entity.source !== "trigger") continue;
2389
- if (STANDALONE_YEAR_RE.test(trimmed)) continue;
2390
- if (entity.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
2391
- if ((entity.label === "person" || entity.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
2392
- if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
2393
- if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
2394
- filtered.push(entity);
2599
+ const maxLen = MAX_ENTITY_LENGTH[normalized.label];
2600
+ if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
2601
+ if (SECTION_NUMBER_RE.test(trimmed) && normalized.source !== "trigger") continue;
2602
+ if (STANDALONE_YEAR_RE.test(trimmed) && normalized.source !== "trigger") continue;
2603
+ if (fullText && normalized.source !== "trigger" && /^\d/.test(trimmed) && NUMBER_ABBREV_RE.test(fullText.slice(Math.max(0, normalized.start - 10), normalized.start))) continue;
2604
+ if (normalized.label === "registration number" && /^[\p{L}]{1,2}$/u.test(trimmed)) continue;
2605
+ if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
2606
+ if (normalized.label === "person") {
2607
+ const tokens = trimmed.split(/\s+/u);
2608
+ const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "").toLowerCase();
2609
+ if (tokens.length > 1 && last && PERSON_TRAILING_NOUNS.has(last)) continue;
2610
+ }
2611
+ if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
2612
+ if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
2613
+ if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
2614
+ if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
2615
+ if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
2616
+ filtered.push(normalized);
2395
2617
  }
2396
2618
  return filtered;
2397
2619
  };
@@ -2557,6 +2779,8 @@ const loadPersonStopwords = (ctx) => {
2557
2779
  const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
2558
2780
  /** Sync accessor — returns empty set before init. */
2559
2781
  const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
2782
+ const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
2783
+ const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
2560
2784
  /**
2561
2785
  * Resolve which dictionaries to load based on country
2562
2786
  * and category filters, load them, and build the deny
@@ -2583,6 +2807,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
2583
2807
  const ids = [...dataModule.ALL_DICTIONARY_IDS].filter((id) => {
2584
2808
  const meta = dataModule.DICTIONARY_META[id];
2585
2809
  if (!meta) return false;
2810
+ if (!config.enableNameCorpus && meta.category === "Names") return false;
2586
2811
  if (excludeCategories.has(meta.category)) return false;
2587
2812
  if (allowedCountries === null) return true;
2588
2813
  if (meta.country === null) return true;
@@ -2669,20 +2894,22 @@ const buildDenyList = async (config, ctx = defaultContext) => {
2669
2894
  sourceList.push([source]);
2670
2895
  }
2671
2896
  };
2672
- for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
2673
- for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
2674
- for (const title of getNameCorpusTitles(ctx)) {
2675
- const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
2676
- if (norm.length === 0) continue;
2677
- const lower = norm.toLowerCase();
2678
- const existing = patternIndex.get(lower);
2679
- if (existing !== void 0) {
2680
- if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
2681
- } else {
2682
- patternIndex.set(lower, patternList.length);
2683
- patternList.push(norm);
2684
- labelList.push(["person"]);
2685
- sourceList.push(["title"]);
2897
+ if (config.enableNameCorpus && !excludeCategories.has("Names")) {
2898
+ for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
2899
+ for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
2900
+ for (const title of getNameCorpusTitles(ctx)) {
2901
+ const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
2902
+ if (norm.length === 0) continue;
2903
+ const lower = norm.toLowerCase();
2904
+ const existing = patternIndex.get(lower);
2905
+ if (existing !== void 0) {
2906
+ if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
2907
+ } else {
2908
+ patternIndex.set(lower, patternList.length);
2909
+ patternList.push(norm);
2910
+ labelList.push(["person"]);
2911
+ sourceList.push(["title"]);
2912
+ }
2686
2913
  }
2687
2914
  }
2688
2915
  if (patternList.length === 0) return null;
@@ -2782,7 +3009,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
2782
3009
  const prev = chain.at(-1);
2783
3010
  if (!prev) break;
2784
3011
  const gap = fullText.slice(prev.end, next.start);
2785
- if (gap.length > 4 || gap.length === 0 || gap.includes("\n") || gap.includes(" ")) break;
3012
+ const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap(prev.text, gap);
3013
+ if (gap.length > 4 || gap.length === 0 || gap.includes("\n") || gap.includes(" ") || PERSON_CHAIN_BREAK_RE.test(gap) || breaksOnPeriod) break;
2786
3014
  chain.push(next);
2787
3015
  j++;
2788
3016
  }
@@ -2792,7 +3020,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
2792
3020
  if (!first || !last) continue;
2793
3021
  const extended = extendPersonName(fullText, first.start, last.end, ctx);
2794
3022
  const score = chain.length >= 2 ? .9 : .5;
2795
- if (chain.length === 1 && isSentenceStart(fullText, first.start)) {
3023
+ if (chain.length === 1) {
2796
3024
  const afterEnd = last.end;
2797
3025
  const rest = fullText.slice(afterEnd).trimStart();
2798
3026
  if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
@@ -3687,6 +3915,25 @@ const initHotwordRules = async () => {
3687
3915
  return initPromise;
3688
3916
  };
3689
3917
  /**
3918
+ * Expand requested output labels with any source labels
3919
+ * that hotword rules may reclassify into them.
3920
+ *
3921
+ * Example: requesting only "date of birth" still needs
3922
+ * "date" candidates to survive until the hotword pass.
3923
+ * If rules are not initialized, or if no labels were
3924
+ * requested, returns the input labels unchanged.
3925
+ */
3926
+ const expandLabelsForHotwordRules = (requestedLabels) => {
3927
+ if (rules === null || requestedLabels.length === 0) return requestedLabels;
3928
+ const requested = new Set(requestedLabels);
3929
+ const expanded = new Set(requestedLabels);
3930
+ for (const rule of rules) {
3931
+ if (rule.reclassifyTo === void 0 || !requested.has(rule.reclassifyTo)) continue;
3932
+ for (const label of rule.targetLabels) expanded.add(label);
3933
+ }
3934
+ return [...expanded];
3935
+ };
3936
+ /**
3690
3937
  * Apply hotword context rules to detected entities.
3691
3938
  *
3692
3939
  * Scans `fullText` once with a single AC automaton
@@ -3797,7 +4044,8 @@ const WORD_START_STOPS = new Set([
3797
4044
  "(",
3798
4045
  ")",
3799
4046
  "[",
3800
- "]"
4047
+ "]",
4048
+ "&"
3801
4049
  ]);
3802
4050
  /**
3803
4051
  * Find the word-start offset at or before `pos`.
@@ -3826,7 +4074,8 @@ const WORD_END_STOPS = new Set([
3826
4074
  "(",
3827
4075
  ")",
3828
4076
  "[",
3829
- "]"
4077
+ "]",
4078
+ "&"
3830
4079
  ]);
3831
4080
  /**
3832
4081
  * Find the word-end offset at or after `pos`.
@@ -4056,8 +4305,9 @@ const enforceBoundaryConsistency = (entities, fullText) => {
4056
4305
  //#endregion
4057
4306
  //#region src/build-unified-search.ts
4058
4307
  const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
4308
+ const legalFormsEnabled = isLegalFormsEnabled(config);
4059
4309
  const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
4060
- buildLegalFormPatterns(),
4310
+ legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
4061
4311
  config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
4062
4312
  patterns: [],
4063
4313
  rules: []
@@ -4288,7 +4538,7 @@ const COLON_LABELS = new Set(["ip address", "mac address"]);
4288
4538
  /** Strip leading/trailing whitespace and punctuation. */
4289
4539
  const sanitizeEntities = (entities) => entities.flatMap((e) => {
4290
4540
  const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
4291
- const leadTrimmed = e.text.replace(new RegExp(`^${strip.source}`, strip.flags), "");
4541
+ const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
4292
4542
  const lead = e.text.length - leadTrimmed.length;
4293
4543
  const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
4294
4544
  if (cleaned.length === 0) return [];
@@ -4345,12 +4595,24 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
4345
4595
  text: fullText.slice(e.start, newEnd)
4346
4596
  };
4347
4597
  });
4598
+ const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
4599
+ const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
4600
+ const filterAllowedLabels = (entities, allowedLabels) => {
4601
+ if (!allowedLabels) return entities;
4602
+ return entities.filter((e) => allowedLabels.has(e.label));
4603
+ };
4604
+ const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
4605
+ const getRequestedNerLabels = (config, expandForHotwords = false) => {
4606
+ const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
4607
+ return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
4608
+ };
4348
4609
  const checkAbort = (signal) => {
4349
4610
  if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
4350
4611
  };
4351
4612
  const configKey = (config, gazetteerEntries) => {
4613
+ const legalFormsEnabled = isLegalFormsEnabled(config);
4352
4614
  const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
4353
- return `${config.enableDenyList}:${config.enableTriggerPhrases}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
4615
+ return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
4354
4616
  };
4355
4617
  /**
4356
4618
  * Get or build a cached search instance. Cache state
@@ -4387,6 +4649,8 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
4387
4649
  const runPipeline = async (options) => {
4388
4650
  const { fullText, config, gazetteerEntries, nerInference = null, onProgress, cachedSearch, signal, context } = options;
4389
4651
  const ctx = context ?? defaultContext;
4652
+ const allowedLabels = createAllowedLabelSet(config);
4653
+ const legalFormsEnabled = isLegalFormsEnabled(config);
4390
4654
  const log = (step, detail) => {
4391
4655
  onProgress?.(step, detail);
4392
4656
  };
@@ -4427,44 +4691,57 @@ const runPipeline = async (options) => {
4427
4691
  if (zones.length > 0) log("zones", [...new Set(zones.map((z) => z.zone))].join(", "));
4428
4692
  }
4429
4693
  checkAbort(signal);
4694
+ const hotwordsActive = enableHotwords && hotwordInitOk;
4695
+ const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
4430
4696
  const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
4431
4697
  checkAbort(signal);
4432
4698
  const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
4433
4699
  const { slices } = search;
4434
- const regexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
4700
+ const rawRegexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
4701
+ const regexEntities = filterAllowedLabels(rawRegexEntities, preHotwordAllowedLabels);
4435
4702
  if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
4436
- const legalFormEntities = processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText);
4703
+ const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
4704
+ const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
4437
4705
  if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
4438
- const triggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
4706
+ const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
4707
+ const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
4439
4708
  if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
4440
4709
  checkAbort(signal);
4710
+ let rawNameCorpusEntities = [];
4441
4711
  let nameCorpusEntities = [];
4442
4712
  if (config.enableNameCorpus && !config.enableDenyList) {
4443
4713
  await initNameCorpus(ctx);
4444
4714
  checkAbort(signal);
4445
- nameCorpusEntities = detectNameCorpus(fullText, ctx);
4715
+ rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
4716
+ nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
4446
4717
  log("name-corpus", `${nameCorpusEntities.length} matches`);
4447
4718
  }
4448
- const denyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
4719
+ const rawDenyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
4720
+ const denyListEntities = filterAllowedLabels(rawDenyListEntities, preHotwordAllowedLabels);
4449
4721
  if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
4450
- const gazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
4722
+ const rawGazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
4723
+ const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
4451
4724
  if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
4452
4725
  checkAbort(signal);
4726
+ const ruleContextEntities = [
4727
+ ...rawTriggerEntities,
4728
+ ...rawRegexEntities,
4729
+ ...rawLegalFormEntities,
4730
+ ...rawNameCorpusEntities,
4731
+ ...rawDenyListEntities,
4732
+ ...rawGazetteerEntities
4733
+ ];
4734
+ let rawNerEntities = [];
4453
4735
  let nerEntities = [];
4454
4736
  if (config.enableNer && nerInference) {
4455
- const maskResult = maskDetectedSpans(fullText, [
4456
- ...triggerEntities,
4457
- ...regexEntities,
4458
- ...legalFormEntities,
4459
- ...nameCorpusEntities,
4460
- ...denyListEntities,
4461
- ...gazetteerEntities
4462
- ]);
4737
+ const maskResult = maskDetectedSpans(fullText, ruleContextEntities);
4463
4738
  log("ner", "running inference...");
4464
- const rawNer = await nerInference(maskResult.maskedText, config.labels, config.threshold, signal);
4465
- nerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
4466
- const dropped = rawNer.length - nerEntities.length;
4467
- log("ner", `${nerEntities.length} entities` + (dropped > 0 ? ` (${dropped} masked)` : ""));
4739
+ const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
4740
+ rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
4741
+ nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
4742
+ const masked = rawNer.length - rawNerEntities.length;
4743
+ const labelFiltered = rawNerEntities.length - nerEntities.length;
4744
+ log("ner", `${nerEntities.length} entities` + (masked > 0 ? ` (${masked} masked)` : "") + (labelFiltered > 0 ? ` (${labelFiltered} label-filtered)` : ""));
4468
4745
  }
4469
4746
  checkAbort(signal);
4470
4747
  const preAddressEntities = [
@@ -4476,23 +4753,23 @@ const runPipeline = async (options) => {
4476
4753
  ...gazetteerEntities,
4477
4754
  ...nerEntities
4478
4755
  ];
4479
- const addressSeedEntities = await processAddressSeeds(literalMatches, slices.streetTypes.start, slices.streetTypes.end, fullText, preAddressEntities);
4756
+ const addressSeedEntities = labelIsAllowed("address", allowedLabels) ? await processAddressSeeds(literalMatches, slices.streetTypes.start, slices.streetTypes.end, fullText, [...ruleContextEntities, ...rawNerEntities]) : [];
4480
4757
  if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
4481
4758
  checkAbort(signal);
4482
4759
  const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
4483
- const preBoostEntities = enableHotwords && hotwordInitOk ? applyHotwordRules(zoneAdjusted, fullText) : zoneAdjusted;
4760
+ const preBoostEntities = hotwordsActive ? filterAllowedLabels(applyHotwordRules(zoneAdjusted, fullText), allowedLabels) : zoneAdjusted;
4484
4761
  let allEntities;
4485
4762
  if (config.enableConfidenceBoost) {
4486
4763
  allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
4487
4764
  const boosted = allEntities.length - preBoostEntities.filter((e) => e.score >= config.threshold).length;
4488
4765
  if (boosted > 0) log("confidence-boost", `${boosted} near-miss promoted`);
4489
4766
  } else allEntities = preBoostEntities.filter((e) => e.score >= config.threshold);
4490
- const streetPatterns = detectStreetPatternsNearAddresses(fullText, allEntities);
4767
+ const streetPatterns = labelIsAllowed("address", allowedLabels) ? detectStreetPatternsNearAddresses(fullText, allEntities) : [];
4491
4768
  if (streetPatterns.length > 0) {
4492
4769
  allEntities = [...allEntities, ...streetPatterns];
4493
4770
  log("street-context", `${streetPatterns.length} street patterns near addresses`);
4494
4771
  }
4495
- const orphanStreets = detectOrphanStreetLines(fullText, allEntities);
4772
+ const orphanStreets = labelIsAllowed("address", allowedLabels) ? detectOrphanStreetLines(fullText, allEntities) : [];
4496
4773
  if (orphanStreets.length > 0) {
4497
4774
  allEntities = [...allEntities, ...orphanStreets];
4498
4775
  log("orphan-streets", `${orphanStreets.length} header street lines`);
@@ -4503,14 +4780,14 @@ const runPipeline = async (options) => {
4503
4780
  const consistent = enforceBoundaryConsistency(mergedExtended, fullText);
4504
4781
  if (consistent.length < mergedExtended.length) log("boundary", `${mergedExtended.length - consistent.length} consolidated`);
4505
4782
  let postOrgEntities = consistent;
4506
- if (config.enableCoreference) {
4783
+ if (config.enableCoreference && labelIsAllowed("organization", allowedLabels)) {
4507
4784
  const thresholded = propagateOrgNames(consistent, fullText).filter((e) => e.score >= config.threshold);
4508
4785
  if (thresholded.length > 0) {
4509
4786
  postOrgEntities = mergeAndDedup(consistent, thresholded);
4510
4787
  log("org-propagation", `${thresholded.length} base names`);
4511
4788
  }
4512
4789
  }
4513
- const merged = filterFalsePositives(postOrgEntities, ctx);
4790
+ const merged = filterFalsePositives(postOrgEntities, ctx, fullText);
4514
4791
  if (merged.length < postOrgEntities.length) log("filter", `removed ${postOrgEntities.length - merged.length} FPs`);
4515
4792
  checkAbort(signal);
4516
4793
  ctx.corefSourceMap.clear();
@@ -4521,11 +4798,11 @@ const runPipeline = async (options) => {
4521
4798
  const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
4522
4799
  if (corefSpans.length > 0) {
4523
4800
  log("coreference-rescan", `${corefSpans.length} aliases`);
4524
- return sanitizeEntities(filterFalsePositives(enforceBoundaryConsistency(mergeAndDedup(merged, corefSpans), fullText), ctx));
4801
+ return sanitizeEntities(filterAllowedLabels(filterFalsePositives(enforceBoundaryConsistency(mergeAndDedup(merged, corefSpans), fullText), ctx, fullText), allowedLabels));
4525
4802
  }
4526
4803
  }
4527
4804
  }
4528
- return sanitizeEntities(merged);
4805
+ return sanitizeEntities(filterAllowedLabels(merged, allowedLabels));
4529
4806
  };
4530
4807
  const OPERATOR_REGISTRY = {
4531
4808
  replace: {