@stll/anonymize 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +44 -48
- package/dist/index.d.mts +20 -2
- package/dist/index.mjs +356 -79
- package/dist/index.mjs.map +1 -1
- package/package.json +13 -9
package/dist/index.mjs
CHANGED
|
@@ -73,6 +73,7 @@ const DEFAULT_ENTITY_LABELS = [
|
|
|
73
73
|
"monetary amount",
|
|
74
74
|
"land parcel"
|
|
75
75
|
];
|
|
76
|
+
const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
76
77
|
//#endregion
|
|
77
78
|
//#region src/context.ts
|
|
78
79
|
/**
|
|
@@ -293,6 +294,33 @@ const getDefinitionPatterns = async (ctx) => {
|
|
|
293
294
|
};
|
|
294
295
|
const SEARCH_WINDOW = 200;
|
|
295
296
|
/**
|
|
297
|
+
* Check whether an alias has textual similarity to
|
|
298
|
+
* the source entity. Prevents roles and structural
|
|
299
|
+
* terms from being treated as name aliases.
|
|
300
|
+
*
|
|
301
|
+
* Three checks (any passes → similar):
|
|
302
|
+
* 1. Word overlap: a word in the alias appears in the
|
|
303
|
+
* entity (case-insensitive, min 2 chars)
|
|
304
|
+
* 2. Initials: alias letters match first letters of
|
|
305
|
+
* entity words ("TB" ↔ "Tomas Bata")
|
|
306
|
+
* 3. Substring: alias is a substring of the entity
|
|
307
|
+
* or vice versa (min 3 chars)
|
|
308
|
+
*/
|
|
309
|
+
const hasEntitySimilarity = (alias, entityText) => {
|
|
310
|
+
const aliasLower = alias.toLowerCase();
|
|
311
|
+
const entityLower = entityText.toLowerCase();
|
|
312
|
+
if (aliasLower.length >= 3 && entityLower.includes(aliasLower)) return true;
|
|
313
|
+
if (entityLower.length >= 3 && aliasLower.includes(entityLower)) return true;
|
|
314
|
+
const aliasWords = aliasLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
315
|
+
const entityWords = entityLower.split(/[\s.,;:'"()/-]+/).filter((w) => w.length >= 2);
|
|
316
|
+
const entityWordSet = new Set(entityWords);
|
|
317
|
+
for (const word of aliasWords) if (entityWordSet.has(word)) return true;
|
|
318
|
+
if (/^[\p{Lu}]+$/u.test(alias) && alias.length >= 2 && alias.length <= entityWords.length) {
|
|
319
|
+
for (let start = 0; start <= entityWords.length - alias.length; start++) if (entityWords.slice(start, start + alias.length).map((w) => w.charAt(0)).join("") === aliasLower) return true;
|
|
320
|
+
}
|
|
321
|
+
return false;
|
|
322
|
+
};
|
|
323
|
+
/**
|
|
296
324
|
* Scan for defined-term patterns near known entities.
|
|
297
325
|
*
|
|
298
326
|
* Legal documents universally follow:
|
|
@@ -332,6 +360,9 @@ const extractDefinedTerms = async (fullText, entities, ctx = defaultContext) =>
|
|
|
332
360
|
break;
|
|
333
361
|
}
|
|
334
362
|
if (bestEntity === null) continue;
|
|
363
|
+
const gapText = fullText.slice(bestEntity.end, defPos);
|
|
364
|
+
if (/(?:;|\.(?=\s*(?:["'„‚(]*\p{Lu}|$)))/u.test(gapText)) continue;
|
|
365
|
+
if (!hasEntitySimilarity(alias, bestEntity.text)) continue;
|
|
335
366
|
const key = `${alias.toLowerCase()}::${bestEntity.label}`;
|
|
336
367
|
if (seen.has(key)) continue;
|
|
337
368
|
seen.add(key);
|
|
@@ -697,6 +728,8 @@ const TOKEN_TYPE = {
|
|
|
697
728
|
CAPITALIZED: "capitalized",
|
|
698
729
|
OTHER: "other"
|
|
699
730
|
};
|
|
731
|
+
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
732
|
+
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
700
733
|
/**
|
|
701
734
|
* Check if a token is in the first-name set, either
|
|
702
735
|
* directly or after stripping Czech/Slovak inflection.
|
|
@@ -828,7 +861,9 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
828
861
|
if (!next) break;
|
|
829
862
|
const prev = chain.at(-1);
|
|
830
863
|
if (prev) {
|
|
831
|
-
|
|
864
|
+
const gap = fullText.slice(prev.end, next.start);
|
|
865
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
|
|
866
|
+
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
832
867
|
}
|
|
833
868
|
if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
|
|
834
869
|
chain.push(next);
|
|
@@ -1396,6 +1431,7 @@ const buildDatePatternsFromMonths = (alt) => {
|
|
|
1396
1431
|
return [
|
|
1397
1432
|
`(?i)\\b\\d{1,2}\\.?\\s+(?:${alt})\\.?\\s+\\d{4}(?:\\s+\\d{1,2}:\\d{2}(?::\\d{2})?)?\\b`,
|
|
1398
1433
|
`(?i)\\b(?:${alt})\\.?\\s+\\d{1,2},?\\s+\\d{4}\\b`,
|
|
1434
|
+
`(?i)\\b(?:${alt})\\.?\\s+\\d{1,2}(?=\\s|[.,;!?)]|$)`,
|
|
1399
1435
|
`(?i)\\b\\d{1,2}(?:st|nd|rd|th)\\s+(?:${alt})\\.?(?:\\s+\\d{4})?(?=\\s|[.,;!?)]|$)`,
|
|
1400
1436
|
`(?i)\\b(?:${alt})\\.?\\s+\\d{4}\\b`,
|
|
1401
1437
|
`(?i)\\b\\d{4}\\.\\s+(?:${alt})\\.?\\s+\\d{1,2}\\.?(?=\\s|[.,;!?)]|$)`,
|
|
@@ -1462,9 +1498,11 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1462
1498
|
if (!symbols && !trailingAlt) return [];
|
|
1463
1499
|
const NUM = "(?:\\d{1,3}(?:[,.'[^\\S\\n\\t]]\\d{3})+|\\d{1,9})";
|
|
1464
1500
|
const patterns = [];
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
if (
|
|
1501
|
+
const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
|
|
1502
|
+
const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
|
|
1503
|
+
if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${END}`);
|
|
1504
|
+
if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${END}`);
|
|
1505
|
+
if (trailingAlt) patterns.push(`\\b${NUM}${DECIMAL}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${END}`);
|
|
1468
1506
|
return patterns;
|
|
1469
1507
|
};
|
|
1470
1508
|
/** Cached promise for currency patterns. Loaded once. */
|
|
@@ -1572,10 +1610,10 @@ const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
|
1572
1610
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
1573
1611
|
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
1574
1612
|
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1575
|
-
const buildPatternString = (forms
|
|
1613
|
+
const buildPatternString = (forms) => {
|
|
1576
1614
|
if (forms.length === 0) return null;
|
|
1577
1615
|
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1578
|
-
return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}
|
|
1616
|
+
return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|${`\\s+(?:a|and|und|et|e|y|i)\\s+(?=[${LOWER}])`})`}(?:${ANY_WORD})){0,10}`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
1579
1617
|
};
|
|
1580
1618
|
/**
|
|
1581
1619
|
* Build legal form regex pattern strings.
|
|
@@ -1600,15 +1638,82 @@ const buildLegalFormPatterns = async () => {
|
|
|
1600
1638
|
}
|
|
1601
1639
|
}
|
|
1602
1640
|
const patterns = [];
|
|
1603
|
-
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f))
|
|
1641
|
+
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
1604
1642
|
if (longPattern) patterns.push(longPattern);
|
|
1605
|
-
const shortPattern = buildPatternString(allForms.filter(isShortForm)
|
|
1643
|
+
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1606
1644
|
if (shortPattern) patterns.push(shortPattern);
|
|
1607
1645
|
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1608
1646
|
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1609
1647
|
patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1610
1648
|
return patterns;
|
|
1611
1649
|
};
|
|
1650
|
+
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
1651
|
+
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
1652
|
+
/**
|
|
1653
|
+
* Find the word ending just before `pos` in `text`,
|
|
1654
|
+
* skipping any whitespace (not newlines).
|
|
1655
|
+
* Returns null if no word is found (e.g., at start
|
|
1656
|
+
* of text, or preceded by non-word chars like ".").
|
|
1657
|
+
*/
|
|
1658
|
+
const findWordBefore = (text, pos) => {
|
|
1659
|
+
let scan = pos - 1;
|
|
1660
|
+
while (scan >= 0) {
|
|
1661
|
+
const ch = text.charAt(scan);
|
|
1662
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1663
|
+
scan--;
|
|
1664
|
+
}
|
|
1665
|
+
if (scan < 0 || text.charAt(scan) === "\n") return null;
|
|
1666
|
+
const wordEnd = scan + 1;
|
|
1667
|
+
while (scan >= 0 && /[\p{L}\p{M}&]/u.test(text.charAt(scan))) scan--;
|
|
1668
|
+
const wordStart = scan + 1;
|
|
1669
|
+
const word = text.slice(wordStart, wordEnd);
|
|
1670
|
+
if (word.length === 0) return null;
|
|
1671
|
+
return {
|
|
1672
|
+
word,
|
|
1673
|
+
start: wordStart
|
|
1674
|
+
};
|
|
1675
|
+
};
|
|
1676
|
+
/**
|
|
1677
|
+
* Extend a match backward through uppercase words and
|
|
1678
|
+
* lowercase connectors. Stops at start of text,
|
|
1679
|
+
* newline, or a word that doesn't qualify.
|
|
1680
|
+
*
|
|
1681
|
+
* Connectors (a, and, und, et, ...) are only consumed
|
|
1682
|
+
* when there is a valid word before them — a trailing
|
|
1683
|
+
* connector at an entity boundary is not consumed.
|
|
1684
|
+
*/
|
|
1685
|
+
const extendBackward = (fullText, matchStart) => {
|
|
1686
|
+
let pos = matchStart;
|
|
1687
|
+
while (pos > 0) {
|
|
1688
|
+
const found = findWordBefore(fullText, pos);
|
|
1689
|
+
if (!found) break;
|
|
1690
|
+
const { word, start: wordStart } = found;
|
|
1691
|
+
const isUpper = /^\p{Lu}/u.test(word);
|
|
1692
|
+
const isConnector = CONNECTOR_RE.test(word);
|
|
1693
|
+
if (isUpper) pos = wordStart;
|
|
1694
|
+
else if (isConnector) {
|
|
1695
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1696
|
+
if (!prev) break;
|
|
1697
|
+
if (!/^\p{Lu}/u.test(prev.word)) break;
|
|
1698
|
+
pos = prev.start;
|
|
1699
|
+
} else break;
|
|
1700
|
+
}
|
|
1701
|
+
return pos;
|
|
1702
|
+
};
|
|
1703
|
+
const trimLeadingClause = (text) => {
|
|
1704
|
+
let cut = -1;
|
|
1705
|
+
for (const match of text.matchAll(LEADING_CLAUSE_RE)) cut = match.index + match[0].length;
|
|
1706
|
+
if (cut <= 0) return {
|
|
1707
|
+
offset: 0,
|
|
1708
|
+
text
|
|
1709
|
+
};
|
|
1710
|
+
const trimmed = text.slice(cut);
|
|
1711
|
+
const leadingWs = trimmed.match(/^\s*/u)?.[0].length ?? 0;
|
|
1712
|
+
return {
|
|
1713
|
+
offset: cut + leadingWs,
|
|
1714
|
+
text: trimmed.slice(leadingWs)
|
|
1715
|
+
};
|
|
1716
|
+
};
|
|
1612
1717
|
/**
|
|
1613
1718
|
* Process legal form matches from the unified search.
|
|
1614
1719
|
* Receives all matches; filters to the legal forms
|
|
@@ -1622,27 +1727,52 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1622
1727
|
const text = match.text.trimEnd();
|
|
1623
1728
|
if (text.length < 5) continue;
|
|
1624
1729
|
if (text.includes("\n")) continue;
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1730
|
+
let entityStart = match.start;
|
|
1731
|
+
let entityText = text;
|
|
1732
|
+
if (fullText) {
|
|
1733
|
+
const extended = extendBackward(fullText, match.start);
|
|
1734
|
+
if (extended < match.start) {
|
|
1735
|
+
entityStart = extended;
|
|
1736
|
+
entityText = fullText.slice(extended, match.start + text.length).trimEnd();
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1740
|
+
if (clauseTrim.offset > 0) {
|
|
1741
|
+
entityStart += clauseTrim.offset;
|
|
1742
|
+
entityText = clauseTrim.text;
|
|
1743
|
+
}
|
|
1744
|
+
const getPrefixInfo = (value) => {
|
|
1745
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1746
|
+
return {
|
|
1747
|
+
prefixEnd,
|
|
1748
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1749
|
+
};
|
|
1750
|
+
};
|
|
1751
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1752
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1628
1753
|
if (isAllCapsMatch && fullText) {
|
|
1629
|
-
const lineStart = fullText.lastIndexOf("\n",
|
|
1630
|
-
const lineEnd = fullText.indexOf("\n",
|
|
1754
|
+
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1755
|
+
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1631
1756
|
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1632
1757
|
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1633
1758
|
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1634
|
-
if ((prefixPart.length > 0 ?
|
|
1759
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1760
|
+
entityStart = match.start;
|
|
1761
|
+
entityText = text;
|
|
1762
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1763
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1764
|
+
}
|
|
1635
1765
|
} else if (isAllCapsMatch) continue;
|
|
1636
|
-
const lastSpace =
|
|
1637
|
-
const rawSuffix = lastSpace !== -1 ?
|
|
1766
|
+
const lastSpace = entityText.lastIndexOf(" ");
|
|
1767
|
+
const rawSuffix = lastSpace !== -1 ? entityText.slice(lastSpace + 1) : "";
|
|
1638
1768
|
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1639
1769
|
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1640
|
-
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(
|
|
1770
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(entityText.slice(0, lastSpace !== -1 ? lastSpace : entityText.length))) continue;
|
|
1641
1771
|
results.push({
|
|
1642
|
-
start:
|
|
1643
|
-
end:
|
|
1772
|
+
start: entityStart,
|
|
1773
|
+
end: entityStart + entityText.length,
|
|
1644
1774
|
label: "organization",
|
|
1645
|
-
text,
|
|
1775
|
+
text: entityText,
|
|
1646
1776
|
score: .95,
|
|
1647
1777
|
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1648
1778
|
});
|
|
@@ -1818,6 +1948,33 @@ const buildTriggerPatterns = async () => {
|
|
|
1818
1948
|
} catch (err) {
|
|
1819
1949
|
if (!(err instanceof Error) || !err.message.includes("Cannot find module")) throw err;
|
|
1820
1950
|
}
|
|
1951
|
+
try {
|
|
1952
|
+
const yearMod = await import("@stll/anonymize-data/config/year-words.json");
|
|
1953
|
+
const data = yearMod.default ?? yearMod;
|
|
1954
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1955
|
+
const yearValidation = compileValidations([{
|
|
1956
|
+
type: "matches-pattern",
|
|
1957
|
+
pattern: "^(?:19|20)\\d{2}\\.?$"
|
|
1958
|
+
}]);
|
|
1959
|
+
for (const [key, words] of Object.entries(data)) {
|
|
1960
|
+
if (key.startsWith("_") || !Array.isArray(words)) continue;
|
|
1961
|
+
for (const word of words) {
|
|
1962
|
+
const lc = word.toLowerCase();
|
|
1963
|
+
if (seen.has(lc)) continue;
|
|
1964
|
+
seen.add(lc);
|
|
1965
|
+
rules.push({
|
|
1966
|
+
trigger: word,
|
|
1967
|
+
label: "date",
|
|
1968
|
+
strategy: {
|
|
1969
|
+
type: "n-words",
|
|
1970
|
+
count: 1
|
|
1971
|
+
},
|
|
1972
|
+
validations: yearValidation,
|
|
1973
|
+
includeTrigger: false
|
|
1974
|
+
});
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
} catch {}
|
|
1821
1978
|
const seen = /* @__PURE__ */ new Map();
|
|
1822
1979
|
for (const rule of rules) {
|
|
1823
1980
|
const key = rule.trigger.toLowerCase();
|
|
@@ -1851,7 +2008,11 @@ const stripQuotes = (value) => {
|
|
|
1851
2008
|
};
|
|
1852
2009
|
};
|
|
1853
2010
|
/** Hard stop characters for to-next-comma scanning. */
|
|
1854
|
-
const COMMA_STOP_CHARS = new Set([
|
|
2011
|
+
const COMMA_STOP_CHARS = new Set([
|
|
2012
|
+
"\n",
|
|
2013
|
+
"(",
|
|
2014
|
+
" "
|
|
2015
|
+
]);
|
|
1855
2016
|
/**
|
|
1856
2017
|
* Field-label keywords that terminate address scanning.
|
|
1857
2018
|
* When a comma in the address strategy is followed by
|
|
@@ -2344,6 +2505,56 @@ const MAX_ENTITY_LENGTH = {
|
|
|
2344
2505
|
};
|
|
2345
2506
|
const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
|
|
2346
2507
|
const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
|
|
2508
|
+
const NUMBER_ABBREV_RE = /(?:^|[\s(])(?:č|čís|nr|no|n)\.\s*$/i;
|
|
2509
|
+
const SIGNING_CLAUSE_ADDRESS_RE = /^(?:v|ve)\s+[^\d,\n]{1,40},?\s+dne$/iu;
|
|
2510
|
+
const PERSON_TRAILING_NOUNS = new Set([
|
|
2511
|
+
"association",
|
|
2512
|
+
"period",
|
|
2513
|
+
"reform"
|
|
2514
|
+
]);
|
|
2515
|
+
const LEGAL_FORM_HEADING_RE = /\b(?:agreement|amendment|contract|exhibit)\b/iu;
|
|
2516
|
+
const LEADING_ARTIFACT_RE = /^(?:\.\s)+/u;
|
|
2517
|
+
const ADDRESS_ROLE_PREFIX_RE = /^(?:prodávajícího|kupujícího|objednatele|zhotovitele|pronajímatele|dodavatele|odběratele|zaměstnance|zaměstnavatele|nájemce)\s+(?=(?:\p{Lu}|\d|ul\.?|ulice|nám\.?|náměstí|tř\.?|třída|nábř\.?|nábřeží|č\.p\.?|č\.ev\.?|sídliště))/iu;
|
|
2518
|
+
const ADDRESS_INLINE_ABBREV_AFTER_RE = /^(?:\p{Lu}[\p{L}\p{M}]{0,3}\.|ul\.?|nám\.?|tř\.?|nábř\.?|č\.p\.?|č\.ev\.?)/u;
|
|
2519
|
+
const ADDRESS_INLINE_ABBREV_BEFORE_RE = /(?:^|[\s,])(?:st|ave|rd|dr|blvd|ln|hwy|pkwy|cir|ct|pl|sq|ter|trl|ste|apt|bldg|fl|ul|nám|tř|nábř|č\.p|č\.ev)$/iu;
|
|
2520
|
+
const ADDRESS_CONTINUATION_WORD_RE = /^(?:suite|building|floor|unit|apartment|room|tower|wing|block|bldg|ste|apt|fl)\b/iu;
|
|
2521
|
+
const trimTrailingAddressProse = (text) => {
|
|
2522
|
+
for (const match of text.matchAll(/\.(?=\s+\p{Lu})/gu)) {
|
|
2523
|
+
const cutoff = match.index;
|
|
2524
|
+
if (cutoff === void 0) continue;
|
|
2525
|
+
const before = text.slice(0, cutoff);
|
|
2526
|
+
if (!HAS_DIGIT_RE.test(before)) continue;
|
|
2527
|
+
const after = text.slice(cutoff + 1).trimStart();
|
|
2528
|
+
if (after.length < 5 || ADDRESS_INLINE_ABBREV_AFTER_RE.test(after) || ADDRESS_INLINE_ABBREV_BEFORE_RE.test(before.trimEnd()) || ADDRESS_CONTINUATION_WORD_RE.test(after)) continue;
|
|
2529
|
+
return before.trimEnd();
|
|
2530
|
+
}
|
|
2531
|
+
return text;
|
|
2532
|
+
};
|
|
2533
|
+
const normalizeEntity = (entity) => {
|
|
2534
|
+
let start = entity.start;
|
|
2535
|
+
let text = entity.text;
|
|
2536
|
+
const trimLeading = (re) => {
|
|
2537
|
+
const match = re.exec(text);
|
|
2538
|
+
if (!match) return;
|
|
2539
|
+
start += match[0].length;
|
|
2540
|
+
text = text.slice(match[0].length);
|
|
2541
|
+
};
|
|
2542
|
+
trimLeading(LEADING_ARTIFACT_RE);
|
|
2543
|
+
trimLeading(/^\s+/u);
|
|
2544
|
+
if (entity.label === "address") {
|
|
2545
|
+
trimLeading(ADDRESS_ROLE_PREFIX_RE);
|
|
2546
|
+
text = trimTrailingAddressProse(text);
|
|
2547
|
+
}
|
|
2548
|
+
const trailingMatch = /[,\s]+$/u.exec(text);
|
|
2549
|
+
if (trailingMatch) text = text.slice(0, text.length - trailingMatch[0].length);
|
|
2550
|
+
if (text.length === 0) return null;
|
|
2551
|
+
return {
|
|
2552
|
+
...entity,
|
|
2553
|
+
start,
|
|
2554
|
+
end: start + text.length,
|
|
2555
|
+
text
|
|
2556
|
+
};
|
|
2557
|
+
};
|
|
2347
2558
|
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
2348
2559
|
/**
|
|
2349
2560
|
* Load generic-roles.json and cache the result on the
|
|
@@ -2377,21 +2588,32 @@ const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
|
2377
2588
|
* Runs as a post-processing step after all detection
|
|
2378
2589
|
* layers have merged.
|
|
2379
2590
|
*/
|
|
2380
|
-
const filterFalsePositives = (entities, ctx = defaultContext) => {
|
|
2591
|
+
const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
2381
2592
|
const filtered = [];
|
|
2382
2593
|
const roles = getGenericRoles(ctx);
|
|
2383
2594
|
for (const entity of entities) {
|
|
2384
|
-
const
|
|
2595
|
+
const normalized = normalizeEntity(entity);
|
|
2596
|
+
if (!normalized) continue;
|
|
2597
|
+
const trimmed = normalized.text;
|
|
2385
2598
|
if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
|
|
2386
|
-
const maxLen = MAX_ENTITY_LENGTH[
|
|
2387
|
-
if (maxLen && trimmed.length > maxLen &&
|
|
2388
|
-
if (SECTION_NUMBER_RE.test(trimmed) &&
|
|
2389
|
-
if (STANDALONE_YEAR_RE.test(trimmed)) continue;
|
|
2390
|
-
if (
|
|
2391
|
-
if (
|
|
2392
|
-
if (
|
|
2393
|
-
if (
|
|
2394
|
-
|
|
2599
|
+
const maxLen = MAX_ENTITY_LENGTH[normalized.label];
|
|
2600
|
+
if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
|
|
2601
|
+
if (SECTION_NUMBER_RE.test(trimmed) && normalized.source !== "trigger") continue;
|
|
2602
|
+
if (STANDALONE_YEAR_RE.test(trimmed) && normalized.source !== "trigger") continue;
|
|
2603
|
+
if (fullText && normalized.source !== "trigger" && /^\d/.test(trimmed) && NUMBER_ABBREV_RE.test(fullText.slice(Math.max(0, normalized.start - 10), normalized.start))) continue;
|
|
2604
|
+
if (normalized.label === "registration number" && /^[\p{L}]{1,2}$/u.test(trimmed)) continue;
|
|
2605
|
+
if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
|
|
2606
|
+
if (normalized.label === "person") {
|
|
2607
|
+
const tokens = trimmed.split(/\s+/u);
|
|
2608
|
+
const last = tokens.at(-1)?.replace(/[.,;:!?]+$/u, "").toLowerCase();
|
|
2609
|
+
if (tokens.length > 1 && last && PERSON_TRAILING_NOUNS.has(last)) continue;
|
|
2610
|
+
}
|
|
2611
|
+
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
|
|
2612
|
+
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
2613
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
2614
|
+
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
2615
|
+
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
2616
|
+
filtered.push(normalized);
|
|
2395
2617
|
}
|
|
2396
2618
|
return filtered;
|
|
2397
2619
|
};
|
|
@@ -2557,6 +2779,8 @@ const loadPersonStopwords = (ctx) => {
|
|
|
2557
2779
|
const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
|
|
2558
2780
|
/** Sync accessor — returns empty set before init. */
|
|
2559
2781
|
const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
|
|
2782
|
+
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
2783
|
+
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
2560
2784
|
/**
|
|
2561
2785
|
* Resolve which dictionaries to load based on country
|
|
2562
2786
|
* and category filters, load them, and build the deny
|
|
@@ -2583,6 +2807,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
2583
2807
|
const ids = [...dataModule.ALL_DICTIONARY_IDS].filter((id) => {
|
|
2584
2808
|
const meta = dataModule.DICTIONARY_META[id];
|
|
2585
2809
|
if (!meta) return false;
|
|
2810
|
+
if (!config.enableNameCorpus && meta.category === "Names") return false;
|
|
2586
2811
|
if (excludeCategories.has(meta.category)) return false;
|
|
2587
2812
|
if (allowedCountries === null) return true;
|
|
2588
2813
|
if (meta.country === null) return true;
|
|
@@ -2669,20 +2894,22 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
2669
2894
|
sourceList.push([source]);
|
|
2670
2895
|
}
|
|
2671
2896
|
};
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
const
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
if (
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2897
|
+
if (config.enableNameCorpus && !excludeCategories.has("Names")) {
|
|
2898
|
+
for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
|
|
2899
|
+
for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
|
|
2900
|
+
for (const title of getNameCorpusTitles(ctx)) {
|
|
2901
|
+
const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
|
|
2902
|
+
if (norm.length === 0) continue;
|
|
2903
|
+
const lower = norm.toLowerCase();
|
|
2904
|
+
const existing = patternIndex.get(lower);
|
|
2905
|
+
if (existing !== void 0) {
|
|
2906
|
+
if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
|
|
2907
|
+
} else {
|
|
2908
|
+
patternIndex.set(lower, patternList.length);
|
|
2909
|
+
patternList.push(norm);
|
|
2910
|
+
labelList.push(["person"]);
|
|
2911
|
+
sourceList.push(["title"]);
|
|
2912
|
+
}
|
|
2686
2913
|
}
|
|
2687
2914
|
}
|
|
2688
2915
|
if (patternList.length === 0) return null;
|
|
@@ -2782,7 +3009,8 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
2782
3009
|
const prev = chain.at(-1);
|
|
2783
3010
|
if (!prev) break;
|
|
2784
3011
|
const gap = fullText.slice(prev.end, next.start);
|
|
2785
|
-
|
|
3012
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap(prev.text, gap);
|
|
3013
|
+
if (gap.length > 4 || gap.length === 0 || gap.includes("\n") || gap.includes(" ") || PERSON_CHAIN_BREAK_RE.test(gap) || breaksOnPeriod) break;
|
|
2786
3014
|
chain.push(next);
|
|
2787
3015
|
j++;
|
|
2788
3016
|
}
|
|
@@ -2792,7 +3020,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
2792
3020
|
if (!first || !last) continue;
|
|
2793
3021
|
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
2794
3022
|
const score = chain.length >= 2 ? .9 : .5;
|
|
2795
|
-
if (chain.length === 1
|
|
3023
|
+
if (chain.length === 1) {
|
|
2796
3024
|
const afterEnd = last.end;
|
|
2797
3025
|
const rest = fullText.slice(afterEnd).trimStart();
|
|
2798
3026
|
if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
|
|
@@ -3687,6 +3915,25 @@ const initHotwordRules = async () => {
|
|
|
3687
3915
|
return initPromise;
|
|
3688
3916
|
};
|
|
3689
3917
|
/**
|
|
3918
|
+
* Expand requested output labels with any source labels
|
|
3919
|
+
* that hotword rules may reclassify into them.
|
|
3920
|
+
*
|
|
3921
|
+
* Example: requesting only "date of birth" still needs
|
|
3922
|
+
* "date" candidates to survive until the hotword pass.
|
|
3923
|
+
* If rules are not initialized, or if no labels were
|
|
3924
|
+
* requested, returns the input labels unchanged.
|
|
3925
|
+
*/
|
|
3926
|
+
const expandLabelsForHotwordRules = (requestedLabels) => {
|
|
3927
|
+
if (rules === null || requestedLabels.length === 0) return requestedLabels;
|
|
3928
|
+
const requested = new Set(requestedLabels);
|
|
3929
|
+
const expanded = new Set(requestedLabels);
|
|
3930
|
+
for (const rule of rules) {
|
|
3931
|
+
if (rule.reclassifyTo === void 0 || !requested.has(rule.reclassifyTo)) continue;
|
|
3932
|
+
for (const label of rule.targetLabels) expanded.add(label);
|
|
3933
|
+
}
|
|
3934
|
+
return [...expanded];
|
|
3935
|
+
};
|
|
3936
|
+
/**
|
|
3690
3937
|
* Apply hotword context rules to detected entities.
|
|
3691
3938
|
*
|
|
3692
3939
|
* Scans `fullText` once with a single AC automaton
|
|
@@ -3797,7 +4044,8 @@ const WORD_START_STOPS = new Set([
|
|
|
3797
4044
|
"(",
|
|
3798
4045
|
")",
|
|
3799
4046
|
"[",
|
|
3800
|
-
"]"
|
|
4047
|
+
"]",
|
|
4048
|
+
"&"
|
|
3801
4049
|
]);
|
|
3802
4050
|
/**
|
|
3803
4051
|
* Find the word-start offset at or before `pos`.
|
|
@@ -3826,7 +4074,8 @@ const WORD_END_STOPS = new Set([
|
|
|
3826
4074
|
"(",
|
|
3827
4075
|
")",
|
|
3828
4076
|
"[",
|
|
3829
|
-
"]"
|
|
4077
|
+
"]",
|
|
4078
|
+
"&"
|
|
3830
4079
|
]);
|
|
3831
4080
|
/**
|
|
3832
4081
|
* Find the word-end offset at or after `pos`.
|
|
@@ -4056,8 +4305,9 @@ const enforceBoundaryConsistency = (entities, fullText) => {
|
|
|
4056
4305
|
//#endregion
|
|
4057
4306
|
//#region src/build-unified-search.ts
|
|
4058
4307
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
4308
|
+
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
4059
4309
|
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
4060
|
-
buildLegalFormPatterns(),
|
|
4310
|
+
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
4061
4311
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
4062
4312
|
patterns: [],
|
|
4063
4313
|
rules: []
|
|
@@ -4288,7 +4538,7 @@ const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
|
4288
4538
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
4289
4539
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
4290
4540
|
const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
|
|
4291
|
-
const leadTrimmed = e.text.replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
4541
|
+
const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
4292
4542
|
const lead = e.text.length - leadTrimmed.length;
|
|
4293
4543
|
const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
|
|
4294
4544
|
if (cleaned.length === 0) return [];
|
|
@@ -4345,12 +4595,24 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
|
|
|
4345
4595
|
text: fullText.slice(e.start, newEnd)
|
|
4346
4596
|
};
|
|
4347
4597
|
});
|
|
4598
|
+
const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
4599
|
+
const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
|
|
4600
|
+
const filterAllowedLabels = (entities, allowedLabels) => {
|
|
4601
|
+
if (!allowedLabels) return entities;
|
|
4602
|
+
return entities.filter((e) => allowedLabels.has(e.label));
|
|
4603
|
+
};
|
|
4604
|
+
const labelIsAllowed = (label, allowedLabels) => !allowedLabels || allowedLabels.has(label);
|
|
4605
|
+
const getRequestedNerLabels = (config, expandForHotwords = false) => {
|
|
4606
|
+
const labels = config.labels.length > 0 ? config.labels : DEFAULT_ENTITY_LABELS;
|
|
4607
|
+
return expandForHotwords ? expandLabelsForHotwordRules(labels) : labels;
|
|
4608
|
+
};
|
|
4348
4609
|
const checkAbort = (signal) => {
|
|
4349
4610
|
if (signal?.aborted) throw new DOMException("Pipeline aborted", "AbortError");
|
|
4350
4611
|
};
|
|
4351
4612
|
const configKey = (config, gazetteerEntries) => {
|
|
4613
|
+
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
4352
4614
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
4353
|
-
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
4615
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
4354
4616
|
};
|
|
4355
4617
|
/**
|
|
4356
4618
|
* Get or build a cached search instance. Cache state
|
|
@@ -4387,6 +4649,8 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
|
|
|
4387
4649
|
const runPipeline = async (options) => {
|
|
4388
4650
|
const { fullText, config, gazetteerEntries, nerInference = null, onProgress, cachedSearch, signal, context } = options;
|
|
4389
4651
|
const ctx = context ?? defaultContext;
|
|
4652
|
+
const allowedLabels = createAllowedLabelSet(config);
|
|
4653
|
+
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
4390
4654
|
const log = (step, detail) => {
|
|
4391
4655
|
onProgress?.(step, detail);
|
|
4392
4656
|
};
|
|
@@ -4427,44 +4691,57 @@ const runPipeline = async (options) => {
|
|
|
4427
4691
|
if (zones.length > 0) log("zones", [...new Set(zones.map((z) => z.zone))].join(", "));
|
|
4428
4692
|
}
|
|
4429
4693
|
checkAbort(signal);
|
|
4694
|
+
const hotwordsActive = enableHotwords && hotwordInitOk;
|
|
4695
|
+
const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
|
|
4430
4696
|
const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
|
|
4431
4697
|
checkAbort(signal);
|
|
4432
4698
|
const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
4433
4699
|
const { slices } = search;
|
|
4434
|
-
const
|
|
4700
|
+
const rawRegexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
|
|
4701
|
+
const regexEntities = filterAllowedLabels(rawRegexEntities, preHotwordAllowedLabels);
|
|
4435
4702
|
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
4436
|
-
const
|
|
4703
|
+
const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
|
|
4704
|
+
const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
|
|
4437
4705
|
if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
|
|
4438
|
-
const
|
|
4706
|
+
const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
|
|
4707
|
+
const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
|
|
4439
4708
|
if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
|
|
4440
4709
|
checkAbort(signal);
|
|
4710
|
+
let rawNameCorpusEntities = [];
|
|
4441
4711
|
let nameCorpusEntities = [];
|
|
4442
4712
|
if (config.enableNameCorpus && !config.enableDenyList) {
|
|
4443
4713
|
await initNameCorpus(ctx);
|
|
4444
4714
|
checkAbort(signal);
|
|
4445
|
-
|
|
4715
|
+
rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
|
|
4716
|
+
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|
|
4446
4717
|
log("name-corpus", `${nameCorpusEntities.length} matches`);
|
|
4447
4718
|
}
|
|
4448
|
-
const
|
|
4719
|
+
const rawDenyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
|
|
4720
|
+
const denyListEntities = filterAllowedLabels(rawDenyListEntities, preHotwordAllowedLabels);
|
|
4449
4721
|
if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
|
|
4450
|
-
const
|
|
4722
|
+
const rawGazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
|
|
4723
|
+
const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
|
|
4451
4724
|
if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
|
|
4452
4725
|
checkAbort(signal);
|
|
4726
|
+
const ruleContextEntities = [
|
|
4727
|
+
...rawTriggerEntities,
|
|
4728
|
+
...rawRegexEntities,
|
|
4729
|
+
...rawLegalFormEntities,
|
|
4730
|
+
...rawNameCorpusEntities,
|
|
4731
|
+
...rawDenyListEntities,
|
|
4732
|
+
...rawGazetteerEntities
|
|
4733
|
+
];
|
|
4734
|
+
let rawNerEntities = [];
|
|
4453
4735
|
let nerEntities = [];
|
|
4454
4736
|
if (config.enableNer && nerInference) {
|
|
4455
|
-
const maskResult = maskDetectedSpans(fullText,
|
|
4456
|
-
...triggerEntities,
|
|
4457
|
-
...regexEntities,
|
|
4458
|
-
...legalFormEntities,
|
|
4459
|
-
...nameCorpusEntities,
|
|
4460
|
-
...denyListEntities,
|
|
4461
|
-
...gazetteerEntities
|
|
4462
|
-
]);
|
|
4737
|
+
const maskResult = maskDetectedSpans(fullText, ruleContextEntities);
|
|
4463
4738
|
log("ner", "running inference...");
|
|
4464
|
-
const rawNer = await nerInference(maskResult.maskedText, config
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4739
|
+
const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
|
|
4740
|
+
rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
|
|
4741
|
+
nerEntities = filterAllowedLabels(rawNerEntities, preHotwordAllowedLabels);
|
|
4742
|
+
const masked = rawNer.length - rawNerEntities.length;
|
|
4743
|
+
const labelFiltered = rawNerEntities.length - nerEntities.length;
|
|
4744
|
+
log("ner", `${nerEntities.length} entities` + (masked > 0 ? ` (${masked} masked)` : "") + (labelFiltered > 0 ? ` (${labelFiltered} label-filtered)` : ""));
|
|
4468
4745
|
}
|
|
4469
4746
|
checkAbort(signal);
|
|
4470
4747
|
const preAddressEntities = [
|
|
@@ -4476,23 +4753,23 @@ const runPipeline = async (options) => {
|
|
|
4476
4753
|
...gazetteerEntities,
|
|
4477
4754
|
...nerEntities
|
|
4478
4755
|
];
|
|
4479
|
-
const addressSeedEntities = await processAddressSeeds(literalMatches, slices.streetTypes.start, slices.streetTypes.end, fullText,
|
|
4756
|
+
const addressSeedEntities = labelIsAllowed("address", allowedLabels) ? await processAddressSeeds(literalMatches, slices.streetTypes.start, slices.streetTypes.end, fullText, [...ruleContextEntities, ...rawNerEntities]) : [];
|
|
4480
4757
|
if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
|
|
4481
4758
|
checkAbort(signal);
|
|
4482
4759
|
const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
|
|
4483
|
-
const preBoostEntities =
|
|
4760
|
+
const preBoostEntities = hotwordsActive ? filterAllowedLabels(applyHotwordRules(zoneAdjusted, fullText), allowedLabels) : zoneAdjusted;
|
|
4484
4761
|
let allEntities;
|
|
4485
4762
|
if (config.enableConfidenceBoost) {
|
|
4486
4763
|
allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
|
|
4487
4764
|
const boosted = allEntities.length - preBoostEntities.filter((e) => e.score >= config.threshold).length;
|
|
4488
4765
|
if (boosted > 0) log("confidence-boost", `${boosted} near-miss promoted`);
|
|
4489
4766
|
} else allEntities = preBoostEntities.filter((e) => e.score >= config.threshold);
|
|
4490
|
-
const streetPatterns = detectStreetPatternsNearAddresses(fullText, allEntities);
|
|
4767
|
+
const streetPatterns = labelIsAllowed("address", allowedLabels) ? detectStreetPatternsNearAddresses(fullText, allEntities) : [];
|
|
4491
4768
|
if (streetPatterns.length > 0) {
|
|
4492
4769
|
allEntities = [...allEntities, ...streetPatterns];
|
|
4493
4770
|
log("street-context", `${streetPatterns.length} street patterns near addresses`);
|
|
4494
4771
|
}
|
|
4495
|
-
const orphanStreets = detectOrphanStreetLines(fullText, allEntities);
|
|
4772
|
+
const orphanStreets = labelIsAllowed("address", allowedLabels) ? detectOrphanStreetLines(fullText, allEntities) : [];
|
|
4496
4773
|
if (orphanStreets.length > 0) {
|
|
4497
4774
|
allEntities = [...allEntities, ...orphanStreets];
|
|
4498
4775
|
log("orphan-streets", `${orphanStreets.length} header street lines`);
|
|
@@ -4503,14 +4780,14 @@ const runPipeline = async (options) => {
|
|
|
4503
4780
|
const consistent = enforceBoundaryConsistency(mergedExtended, fullText);
|
|
4504
4781
|
if (consistent.length < mergedExtended.length) log("boundary", `${mergedExtended.length - consistent.length} consolidated`);
|
|
4505
4782
|
let postOrgEntities = consistent;
|
|
4506
|
-
if (config.enableCoreference) {
|
|
4783
|
+
if (config.enableCoreference && labelIsAllowed("organization", allowedLabels)) {
|
|
4507
4784
|
const thresholded = propagateOrgNames(consistent, fullText).filter((e) => e.score >= config.threshold);
|
|
4508
4785
|
if (thresholded.length > 0) {
|
|
4509
4786
|
postOrgEntities = mergeAndDedup(consistent, thresholded);
|
|
4510
4787
|
log("org-propagation", `${thresholded.length} base names`);
|
|
4511
4788
|
}
|
|
4512
4789
|
}
|
|
4513
|
-
const merged = filterFalsePositives(postOrgEntities, ctx);
|
|
4790
|
+
const merged = filterFalsePositives(postOrgEntities, ctx, fullText);
|
|
4514
4791
|
if (merged.length < postOrgEntities.length) log("filter", `removed ${postOrgEntities.length - merged.length} FPs`);
|
|
4515
4792
|
checkAbort(signal);
|
|
4516
4793
|
ctx.corefSourceMap.clear();
|
|
@@ -4521,11 +4798,11 @@ const runPipeline = async (options) => {
|
|
|
4521
4798
|
const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
|
|
4522
4799
|
if (corefSpans.length > 0) {
|
|
4523
4800
|
log("coreference-rescan", `${corefSpans.length} aliases`);
|
|
4524
|
-
return sanitizeEntities(filterFalsePositives(enforceBoundaryConsistency(mergeAndDedup(merged, corefSpans), fullText), ctx));
|
|
4801
|
+
return sanitizeEntities(filterAllowedLabels(filterFalsePositives(enforceBoundaryConsistency(mergeAndDedup(merged, corefSpans), fullText), ctx, fullText), allowedLabels));
|
|
4525
4802
|
}
|
|
4526
4803
|
}
|
|
4527
4804
|
}
|
|
4528
|
-
return sanitizeEntities(merged);
|
|
4805
|
+
return sanitizeEntities(filterAllowedLabels(merged, allowedLabels));
|
|
4529
4806
|
};
|
|
4530
4807
|
const OPERATOR_REGISTRY = {
|
|
4531
4808
|
replace: {
|