@stll/anonymize-wasm 1.4.5 → 1.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-boundaries.mjs +2 -0
- package/dist/constants.mjs.map +1 -1
- package/dist/document-structure-headings.mjs +90 -0
- package/dist/document-structure-headings.mjs.map +1 -0
- package/dist/sentence-verb-indicators.mjs +10 -1
- package/dist/triggers.de.mjs +2 -2
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +11 -16
- package/dist/wasm.mjs +1527 -196
- package/dist/wasm.mjs.map +1 -1
- package/package.json +2 -2
- package/dist/address-street-types.mjs +0 -244
- package/dist/address-street-types.mjs.map +0 -1
- package/dist/amount-words.mjs +0 -478
- package/dist/amount-words.mjs.map +0 -1
- package/dist/rolldown-runtime.mjs +0 -13
package/dist/wasm.mjs
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
1
|
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
|
|
2
|
-
import { t as amount_words_default } from "./amount-words.mjs";
|
|
3
|
-
import { t as address_street_types_default } from "./address-street-types.mjs";
|
|
4
2
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
5
|
-
import { at, au, be, bg, br, ch, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
|
|
3
|
+
import { at, au, be, bg, br, ch, cn, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
|
|
6
4
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
5
|
+
//#region \0rolldown/runtime.js
|
|
6
|
+
var __defProp = Object.defineProperty;
|
|
7
|
+
var __exportAll = (all, no_symbols) => {
|
|
8
|
+
let target = {};
|
|
9
|
+
for (var name in all) __defProp(target, name, {
|
|
10
|
+
get: all[name],
|
|
11
|
+
enumerable: true
|
|
12
|
+
});
|
|
13
|
+
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
14
|
+
return target;
|
|
15
|
+
};
|
|
16
|
+
//#endregion
|
|
7
17
|
//#region src/search-engine.ts
|
|
8
18
|
let _TextSearch;
|
|
9
19
|
const initTextSearch = (ctor) => {
|
|
@@ -180,7 +190,7 @@ const loadLanguageConfigs = async (configType, mapFn) => {
|
|
|
180
190
|
}
|
|
181
191
|
return lang[configType] === true;
|
|
182
192
|
}).map(([code]) => code) : [...FALLBACK_LANGUAGES[configType]];
|
|
183
|
-
const results =
|
|
193
|
+
const results = codes.map(() => void 0);
|
|
184
194
|
const loads = codes.map(async (code, i) => {
|
|
185
195
|
const loader = registry[code];
|
|
186
196
|
if (!loader) {
|
|
@@ -897,12 +907,8 @@ const loadSentenceVerbIndicators = async () => {
|
|
|
897
907
|
return sentenceVerbIndicatorsPromise;
|
|
898
908
|
};
|
|
899
909
|
const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTENCE_VERB_INDICATORS_SEED;
|
|
900
|
-
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
901
|
-
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
902
|
-
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
903
|
-
const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
|
|
904
|
-
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
905
910
|
const HSPACE = "(?:[^\\S\\n]|[\xA0 ])";
|
|
911
|
+
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
906
912
|
const LEGAL_LIST_BOUNDARY_RE = new RegExp(`^[,;]${HSPACE}+(?=\\p{Lu}|(?:\\p{Lu}\\.${HSPACE}?){2,})`, "u");
|
|
907
913
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
908
914
|
const EMPTY_LEADING_CLAUSE_TRIMS = {
|
|
@@ -986,7 +992,7 @@ const loadAllLegalSuffixes = async () => {
|
|
|
986
992
|
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
987
993
|
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
988
994
|
allLegalSuffixesPromise = (async () => {
|
|
989
|
-
let data
|
|
995
|
+
let data;
|
|
990
996
|
try {
|
|
991
997
|
data = (await import("./legal-forms.mjs")).default;
|
|
992
998
|
} catch {
|
|
@@ -1106,20 +1112,6 @@ const loadStructuralSingleCapPrefixes = async () => {
|
|
|
1106
1112
|
};
|
|
1107
1113
|
const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
|
|
1108
1114
|
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, `${HSPACE}+`).replace(/\\\./g, `\\.${HSPACE}?`);
|
|
1109
|
-
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1110
|
-
const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1111
|
-
const buildPatternString = (forms) => {
|
|
1112
|
-
if (forms.length === 0) return null;
|
|
1113
|
-
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1114
|
-
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
1115
|
-
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1116
|
-
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
|
|
1117
|
-
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
1118
|
-
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1119
|
-
const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
|
|
1120
|
-
const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
|
|
1121
|
-
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}${`(?:${HSPACE}+|,${HSPACE}*)`}(?:${alt})(?![${LOWER}])`;
|
|
1122
|
-
};
|
|
1123
1115
|
/**
|
|
1124
1116
|
* Build legal form regex pattern strings.
|
|
1125
1117
|
* Returns an array of regex strings for the unified
|
|
@@ -1127,43 +1119,11 @@ const buildPatternString = (forms) => {
|
|
|
1127
1119
|
* installed.
|
|
1128
1120
|
*/
|
|
1129
1121
|
const buildLegalFormPatterns = async () => {
|
|
1130
|
-
|
|
1131
|
-
try {
|
|
1132
|
-
data = (await import("./legal-forms.mjs")).default;
|
|
1133
|
-
} catch {
|
|
1134
|
-
return [];
|
|
1135
|
-
}
|
|
1136
|
-
const allForms = [];
|
|
1137
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1138
|
-
for (const forms of Object.values(data)) for (const form of forms) if (!seen.has(form)) {
|
|
1139
|
-
seen.add(form);
|
|
1140
|
-
allForms.push(form);
|
|
1141
|
-
}
|
|
1142
|
-
for (const form of LEGAL_SUFFIXES) if (!seen.has(form)) {
|
|
1143
|
-
seen.add(form);
|
|
1144
|
-
allForms.push(form);
|
|
1145
|
-
}
|
|
1146
|
-
const patterns = [];
|
|
1147
|
-
const longPattern = buildPatternString(allForms.filter((f) => !isShortForm(f)));
|
|
1148
|
-
if (longPattern) patterns.push(longPattern);
|
|
1149
|
-
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1150
|
-
if (shortPattern) patterns.push(shortPattern);
|
|
1151
|
-
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1152
|
-
const mixedNameSep = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1153
|
-
const capitalizedNoDigitPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}`;
|
|
1154
|
-
patterns.push(`${capitalizedNoDigitPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1155
|
-
const allcapMixedPrefix = `(?:${ALLCAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD}|\\d{1,4})){1,6}`;
|
|
1156
|
-
patterns.push(`${allcapMixedPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1157
|
-
const dottedLineWrapPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}\\.${HSPACE}*\\n${HSPACE}*`;
|
|
1158
|
-
patterns.push(`${dottedLineWrapPrefix}(?:${allcapAlt})(?![${LOWER}])`);
|
|
1159
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1160
|
-
patterns.push(`${allcapPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1161
|
-
patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
|
|
1162
|
-
return patterns;
|
|
1122
|
+
return [];
|
|
1163
1123
|
};
|
|
1164
|
-
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
1165
|
-
const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
|
|
1166
|
-
const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
1124
|
+
const CONNECTOR_RE$1 = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
1125
|
+
const AND_TYPE_CONNECTOR_RE$1 = /^(?:and|und|et)$/i;
|
|
1126
|
+
const UPPER_LETTER_RE$1 = /^\p{Lu}/u;
|
|
1167
1127
|
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1168
1128
|
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1169
1129
|
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
@@ -1201,7 +1161,7 @@ const findWordBefore = (text, pos) => {
|
|
|
1201
1161
|
};
|
|
1202
1162
|
const hasSingleCapPrefixBefore = (fullText, matchStart) => {
|
|
1203
1163
|
const prev = findWordBefore(fullText, matchStart);
|
|
1204
|
-
return prev !== null && prev.word.length === 1 && UPPER_LETTER_RE.test(prev.word);
|
|
1164
|
+
return prev !== null && prev.word.length === 1 && UPPER_LETTER_RE$1.test(prev.word);
|
|
1205
1165
|
};
|
|
1206
1166
|
const isBareSingleCapStructuralInnerMatch = (fullText, matchStart, text) => {
|
|
1207
1167
|
if (!BARE_SINGLE_CAP_LEGAL_FORM_RE.test(text)) return false;
|
|
@@ -1290,12 +1250,12 @@ const hasDisallowedLineBreak = (text) => {
|
|
|
1290
1250
|
}
|
|
1291
1251
|
return false;
|
|
1292
1252
|
};
|
|
1293
|
-
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
1253
|
+
const hasMiddleInitialBefore$1 = (fullText, pos) => {
|
|
1294
1254
|
const previousWord = findWordBefore(fullText, pos);
|
|
1295
1255
|
if (!previousWord) return false;
|
|
1296
1256
|
let scan = previousWord.start - 1;
|
|
1297
1257
|
while (scan >= 0 && (fullText[scan] === " " || fullText[scan] === " ")) scan--;
|
|
1298
|
-
return scan >= 1 && fullText[scan] === "." && UPPER_LETTER_RE.test(fullText[scan - 1] ?? "");
|
|
1258
|
+
return scan >= 1 && fullText[scan] === "." && UPPER_LETTER_RE$1.test(fullText[scan - 1] ?? "");
|
|
1299
1259
|
};
|
|
1300
1260
|
/**
|
|
1301
1261
|
* Count consecutive uppercase-starting words immediately
|
|
@@ -1319,7 +1279,7 @@ const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
|
|
|
1319
1279
|
while (scan > 0) {
|
|
1320
1280
|
const found = findWordBefore(fullText, scan);
|
|
1321
1281
|
if (found) {
|
|
1322
|
-
if (UPPER_LETTER_RE.test(found.word)) {
|
|
1282
|
+
if (UPPER_LETTER_RE$1.test(found.word)) {
|
|
1323
1283
|
count++;
|
|
1324
1284
|
scan = found.start;
|
|
1325
1285
|
continue;
|
|
@@ -1327,7 +1287,7 @@ const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
|
|
|
1327
1287
|
if (crossInNamePreps && IN_NAME_PREPOSITION_RE.test(found.word)) {
|
|
1328
1288
|
const prev = findWordBefore(fullText, found.start);
|
|
1329
1289
|
if (!prev) break;
|
|
1330
|
-
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1290
|
+
if (!UPPER_LETTER_RE$1.test(prev.word)) break;
|
|
1331
1291
|
scan = found.start;
|
|
1332
1292
|
continue;
|
|
1333
1293
|
}
|
|
@@ -1335,7 +1295,7 @@ const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
|
|
|
1335
1295
|
}
|
|
1336
1296
|
let p = scan - 1;
|
|
1337
1297
|
while (p >= 0 && (fullText[p] === " " || fullText[p] === " ")) p--;
|
|
1338
|
-
if (p >= 1 && fullText[p] === "." && UPPER_LETTER_RE.test(fullText[p - 1] ?? "")) {
|
|
1298
|
+
if (p >= 1 && fullText[p] === "." && UPPER_LETTER_RE$1.test(fullText[p - 1] ?? "")) {
|
|
1339
1299
|
count++;
|
|
1340
1300
|
scan = p - 1;
|
|
1341
1301
|
continue;
|
|
@@ -1414,30 +1374,30 @@ const extendBackward = (fullText, matchStart, options = {}) => {
|
|
|
1414
1374
|
const found = findWordBefore(fullText, pos);
|
|
1415
1375
|
if (!found) break;
|
|
1416
1376
|
const { word, start: wordStart } = found;
|
|
1417
|
-
const isUpper = UPPER_LETTER_RE.test(word);
|
|
1418
|
-
const isConnector = CONNECTOR_RE.test(word);
|
|
1377
|
+
const isUpper = UPPER_LETTER_RE$1.test(word);
|
|
1378
|
+
const isConnector = CONNECTOR_RE$1.test(word);
|
|
1419
1379
|
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1420
1380
|
if (isUpper) pos = wordStart;
|
|
1421
|
-
else if (isConnector) if (AND_TYPE_CONNECTOR_RE.test(word)) {
|
|
1381
|
+
else if (isConnector) if (AND_TYPE_CONNECTOR_RE$1.test(word)) {
|
|
1422
1382
|
const prev = findWordBefore(fullText, wordStart);
|
|
1423
1383
|
if (!prev) break;
|
|
1424
|
-
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1384
|
+
if (!UPPER_LETTER_RE$1.test(prev.word)) break;
|
|
1425
1385
|
if (isKnownLegalFormSuffix(prev.word)) break;
|
|
1426
1386
|
const upperWordsBefore = countUpperWordsBefore(fullText, wordStart, suffixMode);
|
|
1427
|
-
const middleInitialBefore = hasMiddleInitialBefore(fullText, wordStart);
|
|
1387
|
+
const middleInitialBefore = hasMiddleInitialBefore$1(fullText, wordStart);
|
|
1428
1388
|
if (upperWordsBefore <= 1 && (getClauseNounHeadsSync().has(prev.word.toLowerCase()) || getConnectorProseHeadsSync().has(prev.word.toLowerCase()))) break;
|
|
1429
1389
|
if (suffixMode ? middleInitialBefore && hasSingleCapPrefixBefore(fullText, matchStart) : upperWordsBefore === 2 && !isInNameLegalFormWord(prev.word) || middleInitialBefore) break;
|
|
1430
1390
|
pos = prev.start;
|
|
1431
1391
|
} else {
|
|
1432
1392
|
const prev = findWordBefore(fullText, wordStart);
|
|
1433
1393
|
if (!prev) break;
|
|
1434
|
-
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1394
|
+
if (!UPPER_LETTER_RE$1.test(prev.word)) break;
|
|
1435
1395
|
pos = prev.start;
|
|
1436
1396
|
}
|
|
1437
1397
|
else if (isInNamePrep) {
|
|
1438
1398
|
const prev = findWordBefore(fullText, wordStart);
|
|
1439
1399
|
if (!prev) break;
|
|
1440
|
-
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1400
|
+
if (!UPPER_LETTER_RE$1.test(prev.word)) break;
|
|
1441
1401
|
pos = prev.start;
|
|
1442
1402
|
} else break;
|
|
1443
1403
|
}
|
|
@@ -1458,13 +1418,18 @@ const trimLeadingClause = (text) => {
|
|
|
1458
1418
|
"amongst",
|
|
1459
1419
|
"between"
|
|
1460
1420
|
]);
|
|
1421
|
+
const verbIndicators = getSentenceVerbIndicatorsSync();
|
|
1461
1422
|
if (directPrefixAlternation.length > 0) {
|
|
1462
1423
|
const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
|
|
1463
1424
|
for (const match of text.matchAll(directPrefixRe)) {
|
|
1464
1425
|
const matchedPrefix = match[0].trim().toLowerCase();
|
|
1465
1426
|
const before = text.slice(0, match.index);
|
|
1466
|
-
if (COMMA_GATED_DIRECT_PREFIXES.has(matchedPrefix)
|
|
1467
|
-
|
|
1427
|
+
if (COMMA_GATED_DIRECT_PREFIXES.has(matchedPrefix)) {
|
|
1428
|
+
const hasComma = /,\s*$/u.test(before);
|
|
1429
|
+
const hasSentenceVerb = (before.match(/\p{L}[\p{L}\p{M}'’]*/gu) ?? []).some((word) => /^\p{Ll}/u.test(word) && verbIndicators.has(word.toLowerCase()));
|
|
1430
|
+
if (!hasComma && !hasSentenceVerb) continue;
|
|
1431
|
+
}
|
|
1432
|
+
const words = before.match(/\p{L}[\p{L}\p{M}'’]*/gu) ?? [];
|
|
1468
1433
|
if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
|
|
1469
1434
|
}
|
|
1470
1435
|
}
|
|
@@ -1501,7 +1466,7 @@ const trimLeadingClause = (text) => {
|
|
|
1501
1466
|
* otherwise the trim falls back to a no-op and sentence-
|
|
1502
1467
|
* fragment fixes do not apply.
|
|
1503
1468
|
*/
|
|
1504
|
-
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) => {
|
|
1469
|
+
const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText, options = {}) => {
|
|
1505
1470
|
const results = [];
|
|
1506
1471
|
for (const match of allMatches) {
|
|
1507
1472
|
const idx = match.pattern;
|
|
@@ -1519,9 +1484,9 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1519
1484
|
if (firstWordMatch !== null && (roleHeads.has(firstWordText.toLowerCase()) || firstWordLeading.length > 0 && roleHeads.has(firstWordLeading.toLowerCase()))) {
|
|
1520
1485
|
let suffixOffset = -1;
|
|
1521
1486
|
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1522
|
-
const
|
|
1523
|
-
if (
|
|
1524
|
-
suffixOffset =
|
|
1487
|
+
const suffixIdx = text.lastIndexOf(suffix);
|
|
1488
|
+
if (suffixIdx !== -1 && suffixIdx + suffix.length >= text.length - 1) {
|
|
1489
|
+
suffixOffset = suffixIdx;
|
|
1525
1490
|
break;
|
|
1526
1491
|
}
|
|
1527
1492
|
}
|
|
@@ -1531,7 +1496,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1531
1496
|
const midSection = text.slice(midStart, midEnd);
|
|
1532
1497
|
const verbIndicators = getSentenceVerbIndicatorsSync();
|
|
1533
1498
|
let lastVerbEndInMid = -1;
|
|
1534
|
-
for (const
|
|
1499
|
+
for (const wordMatch of midSection.matchAll(/(?<![\p{L}\p{N}])[\p{L}\p{M}]+/gu)) if (wordMatch[0] !== void 0 && wordMatch.index !== void 0 && verbIndicators.has(wordMatch[0].toLowerCase())) lastVerbEndInMid = wordMatch.index + wordMatch[0].length;
|
|
1535
1500
|
const digitAfterRole = /^\s+\d+(?:\.|\b)/u.test(midSection);
|
|
1536
1501
|
let appositiveRoleHead = false;
|
|
1537
1502
|
if (!digitAfterRole && lastVerbEndInMid === -1 && fullText) {
|
|
@@ -1562,7 +1527,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1562
1527
|
if (processedText.includes("\n") && hasDisallowedLineBreak(processedText)) continue;
|
|
1563
1528
|
let entityStart = processedStart;
|
|
1564
1529
|
let entityText = processedText;
|
|
1565
|
-
if (fullText && !trimmed) {
|
|
1530
|
+
if (fullText && !trimmed && options.suppressExtendBackward !== true) {
|
|
1566
1531
|
const extended = !BARE_SINGLE_CAP_LEGAL_FORM_RE.test(processedText) ? extendBackward(fullText, processedStart) : processedStart;
|
|
1567
1532
|
if (extended < processedStart) {
|
|
1568
1533
|
entityStart = extended;
|
|
@@ -1594,15 +1559,13 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1594
1559
|
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1595
1560
|
entityStart = match.start;
|
|
1596
1561
|
entityText = text;
|
|
1597
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1598
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1599
1562
|
}
|
|
1600
1563
|
} else if (isAllCapsMatch) continue;
|
|
1601
1564
|
const lastSuffixSeparator = findLastSuffixSeparator(entityText);
|
|
1602
1565
|
const rawSuffix = lastSuffixSeparator !== -1 ? entityText.slice(lastSuffixSeparator + 1) : "";
|
|
1603
1566
|
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1604
1567
|
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1605
|
-
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\
|
|
1568
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\p{ASCII}]/u.test(stripDocxSpaces(entityText.slice(0, lastSuffixSeparator !== -1 ? lastSuffixSeparator : entityText.length)))) continue;
|
|
1606
1569
|
results.push({
|
|
1607
1570
|
start: entityStart,
|
|
1608
1571
|
end: entityStart + entityText.length,
|
|
@@ -1983,31 +1946,57 @@ const tryPrefixExtension = (fullText, start, end) => {
|
|
|
1983
1946
|
};
|
|
1984
1947
|
//#endregion
|
|
1985
1948
|
//#region src/util/normalize.ts
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1949
|
+
/**
|
|
1950
|
+
* Normalize typographic variants for search matching.
|
|
1951
|
+
*
|
|
1952
|
+
* Legal documents (especially Czech/German) use
|
|
1953
|
+
* non-breaking spaces, smart quotes, and en/em dashes
|
|
1954
|
+
* that differ from their ASCII equivalents. Since all
|
|
1955
|
+
* replacements are same-length (single code unit →
|
|
1956
|
+
* single code unit), character offsets remain valid.
|
|
1957
|
+
*
|
|
1958
|
+
* Lives here (application layer) rather than in the
|
|
1959
|
+
* AC library: what to normalize is domain-specific.
|
|
1960
|
+
*
|
|
1961
|
+
* Uses a char-code lookup (`Map<number, number>`) and
|
|
1962
|
+
* `Uint16Array` instead of 7 sequential `replaceAll`
|
|
1963
|
+
* calls. For a 50 KB document this eliminates ~350 KB
|
|
1964
|
+
* of intermediate string allocations.
|
|
1965
|
+
*
|
|
1966
|
+
* When no replaceable characters are present (common
|
|
1967
|
+
* for plain-text inputs), a fast-path scan returns the
|
|
1968
|
+
* original string without any allocation. When special
|
|
1969
|
+
* characters exist, the string is scanned twice: once
|
|
1970
|
+
* to detect, once to build the replacement array.
|
|
1971
|
+
*/
|
|
1972
|
+
const replacementCode = (code) => {
|
|
1973
|
+
switch (code) {
|
|
1974
|
+
case 160:
|
|
1975
|
+
case 8199:
|
|
1976
|
+
case 8239: return 32;
|
|
1977
|
+
case 8211:
|
|
1978
|
+
case 8212: return 45;
|
|
1979
|
+
case 8220:
|
|
1980
|
+
case 8221: return 34;
|
|
1981
|
+
default: return code;
|
|
1982
|
+
}
|
|
1983
|
+
};
|
|
1995
1984
|
/** Chunk size for `String.fromCharCode` to avoid
|
|
1996
1985
|
* hitting the call-stack limit on very large strings. */
|
|
1997
1986
|
const CHUNK_SIZE$1 = 8192;
|
|
1998
1987
|
const normalizeForSearch = (text) => {
|
|
1999
1988
|
let hasSpecial = false;
|
|
2000
|
-
for (let i = 0; i < text.length; i++)
|
|
2001
|
-
|
|
2002
|
-
|
|
1989
|
+
for (let i = 0; i < text.length; i++) {
|
|
1990
|
+
const code = text.charCodeAt(i);
|
|
1991
|
+
if (replacementCode(code) !== code) {
|
|
1992
|
+
hasSpecial = true;
|
|
1993
|
+
break;
|
|
1994
|
+
}
|
|
2003
1995
|
}
|
|
2004
1996
|
if (!hasSpecial) return text;
|
|
2005
1997
|
const len = text.length;
|
|
2006
1998
|
const codes = new Uint16Array(len);
|
|
2007
|
-
for (let i = 0; i < len; i++)
|
|
2008
|
-
const code = text.charCodeAt(i);
|
|
2009
|
-
codes[i] = CHAR_MAP$1.get(code) ?? code;
|
|
2010
|
-
}
|
|
1999
|
+
for (let i = 0; i < len; i++) codes[i] = replacementCode(text.charCodeAt(i));
|
|
2011
2000
|
if (len <= CHUNK_SIZE$1) return String.fromCharCode(...codes);
|
|
2012
2001
|
let result = "";
|
|
2013
2002
|
for (let offset = 0; offset < len; offset += CHUNK_SIZE$1) {
|
|
@@ -7857,6 +7846,7 @@ const NAME_BLOCKLIST = new Set([
|
|
|
7857
7846
|
"island",
|
|
7858
7847
|
"indie"
|
|
7859
7848
|
].map((s) => s.toLowerCase()));
|
|
7849
|
+
let cachedCountryPatterns = null;
|
|
7860
7850
|
/**
|
|
7861
7851
|
* Build country patterns for the literal search instance.
|
|
7862
7852
|
* Returns patterns and parallel metadata arrays.
|
|
@@ -7868,6 +7858,7 @@ const NAME_BLOCKLIST = new Set([
|
|
|
7868
7858
|
* country entry.
|
|
7869
7859
|
*/
|
|
7870
7860
|
const buildCountryPatterns = () => {
|
|
7861
|
+
if (cachedCountryPatterns !== null) return cachedCountryPatterns;
|
|
7871
7862
|
const raw = countries_default;
|
|
7872
7863
|
const surfaceToMeta = /* @__PURE__ */ new Map();
|
|
7873
7864
|
const register = (surface, isoCode, variant) => {
|
|
@@ -7907,7 +7898,7 @@ const buildCountryPatterns = () => {
|
|
|
7907
7898
|
isoCodes.push(isoCode);
|
|
7908
7899
|
variants.push(variant);
|
|
7909
7900
|
}
|
|
7910
|
-
|
|
7901
|
+
cachedCountryPatterns = {
|
|
7911
7902
|
patterns,
|
|
7912
7903
|
data: {
|
|
7913
7904
|
labels,
|
|
@@ -7915,6 +7906,7 @@ const buildCountryPatterns = () => {
|
|
|
7915
7906
|
variants
|
|
7916
7907
|
}
|
|
7917
7908
|
};
|
|
7909
|
+
return cachedCountryPatterns;
|
|
7918
7910
|
};
|
|
7919
7911
|
/**
|
|
7920
7912
|
* Whether the match starts on a proper-noun character.
|
|
@@ -8147,7 +8139,7 @@ const isAllCapsLineNameShaped = (fullText, start) => {
|
|
|
8147
8139
|
const lineEndIdx = fullText.indexOf("\n", start);
|
|
8148
8140
|
const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
|
|
8149
8141
|
if (/\d/.test(line)) return false;
|
|
8150
|
-
const tokens = line.match(/\p{L}[\p{L}\p{M}'
|
|
8142
|
+
const tokens = line.match(/\p{L}[\p{L}\p{M}'-]*/gu) ?? [];
|
|
8151
8143
|
return tokens.length > 0 && tokens.length <= ALL_CAPS_NAME_LINE_MAX_TOKENS;
|
|
8152
8144
|
};
|
|
8153
8145
|
const isAllCapsContextLine = (fullText, start) => {
|
|
@@ -8182,7 +8174,7 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8182
8174
|
if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
|
|
8183
8175
|
const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
|
|
8184
8176
|
const before = fullText.slice(lineStart, start).trimEnd();
|
|
8185
|
-
const lastWord = /\p{L}[\p{L}\p{M}'
|
|
8177
|
+
const lastWord = /\p{L}[\p{L}\p{M}'-]*$/u.exec(before)?.[0];
|
|
8186
8178
|
if (lastWord) {
|
|
8187
8179
|
const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus);
|
|
8188
8180
|
if (lookup(lastWord)) return {
|
|
@@ -8309,7 +8301,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8309
8301
|
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
8310
8302
|
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
8311
8303
|
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
8312
|
-
let score
|
|
8304
|
+
let score;
|
|
8313
8305
|
if (hasTitle && hasCorpusName) score = .95;
|
|
8314
8306
|
else if (corpusCount >= 2) score = .9;
|
|
8315
8307
|
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
@@ -8637,6 +8629,480 @@ const POST_NOMINALS = [
|
|
|
8637
8629
|
"QC"
|
|
8638
8630
|
];
|
|
8639
8631
|
//#endregion
|
|
8632
|
+
//#region src/data/amount-words.json
|
|
8633
|
+
var amount_words_exports = /* @__PURE__ */ __exportAll({
|
|
8634
|
+
_comment: () => _comment$1,
|
|
8635
|
+
default: () => amount_words_default,
|
|
8636
|
+
magnitudeSuffixes: () => magnitudeSuffixes,
|
|
8637
|
+
patterns: () => patterns,
|
|
8638
|
+
percentages: () => percentages,
|
|
8639
|
+
shareQuantityTerms: () => shareQuantityTerms
|
|
8640
|
+
});
|
|
8641
|
+
var _comment$1 = "Patterns for written-out monetary amounts, magnitude suffixes, share-quantity guards, and numeric percentage phrases, by language.";
|
|
8642
|
+
var patterns = [
|
|
8643
|
+
{
|
|
8644
|
+
"lang": "cs",
|
|
8645
|
+
"keywords": ["slovy", "slovně"]
|
|
8646
|
+
},
|
|
8647
|
+
{
|
|
8648
|
+
"lang": "sk",
|
|
8649
|
+
"keywords": ["slovom", "slovami"]
|
|
8650
|
+
},
|
|
8651
|
+
{
|
|
8652
|
+
"lang": "de",
|
|
8653
|
+
"keywords": ["in Worten", "in Buchstaben"]
|
|
8654
|
+
},
|
|
8655
|
+
{
|
|
8656
|
+
"lang": "en",
|
|
8657
|
+
"keywords": ["in words", "in letters"]
|
|
8658
|
+
},
|
|
8659
|
+
{
|
|
8660
|
+
"lang": "fr",
|
|
8661
|
+
"keywords": ["en lettres", "en toutes lettres"]
|
|
8662
|
+
},
|
|
8663
|
+
{
|
|
8664
|
+
"lang": "pl",
|
|
8665
|
+
"keywords": ["słownie"]
|
|
8666
|
+
},
|
|
8667
|
+
{
|
|
8668
|
+
"lang": "it",
|
|
8669
|
+
"keywords": ["in lettere"]
|
|
8670
|
+
},
|
|
8671
|
+
{
|
|
8672
|
+
"lang": "es",
|
|
8673
|
+
"keywords": ["en letras"]
|
|
8674
|
+
}
|
|
8675
|
+
];
|
|
8676
|
+
var percentages = [{
|
|
8677
|
+
"lang": "en",
|
|
8678
|
+
"keywords": ["percent"],
|
|
8679
|
+
"standalone": ["zero", "one hundred"],
|
|
8680
|
+
"ones": [
|
|
8681
|
+
"one",
|
|
8682
|
+
"two",
|
|
8683
|
+
"three",
|
|
8684
|
+
"four",
|
|
8685
|
+
"five",
|
|
8686
|
+
"six",
|
|
8687
|
+
"seven",
|
|
8688
|
+
"eight",
|
|
8689
|
+
"nine"
|
|
8690
|
+
],
|
|
8691
|
+
"teens": [
|
|
8692
|
+
"ten",
|
|
8693
|
+
"eleven",
|
|
8694
|
+
"twelve",
|
|
8695
|
+
"thirteen",
|
|
8696
|
+
"fourteen",
|
|
8697
|
+
"fifteen",
|
|
8698
|
+
"sixteen",
|
|
8699
|
+
"seventeen",
|
|
8700
|
+
"eighteen",
|
|
8701
|
+
"nineteen"
|
|
8702
|
+
],
|
|
8703
|
+
"tens": [
|
|
8704
|
+
"twenty",
|
|
8705
|
+
"thirty",
|
|
8706
|
+
"forty",
|
|
8707
|
+
"fifty",
|
|
8708
|
+
"sixty",
|
|
8709
|
+
"seventy",
|
|
8710
|
+
"eighty",
|
|
8711
|
+
"ninety"
|
|
8712
|
+
],
|
|
8713
|
+
"allowSpaceCompoundSeparator": true
|
|
8714
|
+
}];
|
|
8715
|
+
var magnitudeSuffixes = [
|
|
8716
|
+
{
|
|
8717
|
+
"lang": "cs",
|
|
8718
|
+
"words": [
|
|
8719
|
+
"sto",
|
|
8720
|
+
"sta",
|
|
8721
|
+
"stě",
|
|
8722
|
+
"sty",
|
|
8723
|
+
"stovka",
|
|
8724
|
+
"stovky",
|
|
8725
|
+
"stovek",
|
|
8726
|
+
"tisíc",
|
|
8727
|
+
"tisíce",
|
|
8728
|
+
"tisíců",
|
|
8729
|
+
"milion",
|
|
8730
|
+
"miliony",
|
|
8731
|
+
"milionů",
|
|
8732
|
+
"miliarda",
|
|
8733
|
+
"miliardy",
|
|
8734
|
+
"miliard",
|
|
8735
|
+
"bilion",
|
|
8736
|
+
"biliony",
|
|
8737
|
+
"bilionů"
|
|
8738
|
+
],
|
|
8739
|
+
"abbreviationsCaseInsensitive": ["mld"],
|
|
8740
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8741
|
+
},
|
|
8742
|
+
{
|
|
8743
|
+
"lang": "de",
|
|
8744
|
+
"words": [
|
|
8745
|
+
"hundert",
|
|
8746
|
+
"hunderte",
|
|
8747
|
+
"hunderten",
|
|
8748
|
+
"tausend",
|
|
8749
|
+
"million",
|
|
8750
|
+
"millionen",
|
|
8751
|
+
"milliarde",
|
|
8752
|
+
"milliarden",
|
|
8753
|
+
"billion",
|
|
8754
|
+
"billionen"
|
|
8755
|
+
],
|
|
8756
|
+
"abbreviationsCaseInsensitive": ["mio", "mrd"],
|
|
8757
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8758
|
+
},
|
|
8759
|
+
{
|
|
8760
|
+
"lang": "en",
|
|
8761
|
+
"words": [
|
|
8762
|
+
"hundred",
|
|
8763
|
+
"hundreds",
|
|
8764
|
+
"thousand",
|
|
8765
|
+
"thousands",
|
|
8766
|
+
"million",
|
|
8767
|
+
"millions",
|
|
8768
|
+
"billion",
|
|
8769
|
+
"billions",
|
|
8770
|
+
"trillion",
|
|
8771
|
+
"trillions",
|
|
8772
|
+
"quadrillion",
|
|
8773
|
+
"quadrillions"
|
|
8774
|
+
],
|
|
8775
|
+
"abbreviationsCaseInsensitive": [
|
|
8776
|
+
"bn",
|
|
8777
|
+
"bln",
|
|
8778
|
+
"tn",
|
|
8779
|
+
"trn"
|
|
8780
|
+
],
|
|
8781
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8782
|
+
},
|
|
8783
|
+
{
|
|
8784
|
+
"lang": "es",
|
|
8785
|
+
"words": [
|
|
8786
|
+
"cien",
|
|
8787
|
+
"ciento",
|
|
8788
|
+
"cientos",
|
|
8789
|
+
"millón",
|
|
8790
|
+
"millones",
|
|
8791
|
+
"billón",
|
|
8792
|
+
"billones"
|
|
8793
|
+
],
|
|
8794
|
+
"abbreviationsCaseInsensitive": ["mln"],
|
|
8795
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8796
|
+
},
|
|
8797
|
+
{
|
|
8798
|
+
"lang": "fr",
|
|
8799
|
+
"words": [
|
|
8800
|
+
"cent",
|
|
8801
|
+
"cents",
|
|
8802
|
+
"centaine",
|
|
8803
|
+
"centaines",
|
|
8804
|
+
"mille",
|
|
8805
|
+
"million",
|
|
8806
|
+
"millions",
|
|
8807
|
+
"milliard",
|
|
8808
|
+
"milliards",
|
|
8809
|
+
"billion",
|
|
8810
|
+
"billions"
|
|
8811
|
+
],
|
|
8812
|
+
"abbreviationsCaseInsensitive": [
|
|
8813
|
+
"mio",
|
|
8814
|
+
"md",
|
|
8815
|
+
"mrd"
|
|
8816
|
+
],
|
|
8817
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8818
|
+
},
|
|
8819
|
+
{
|
|
8820
|
+
"lang": "it",
|
|
8821
|
+
"words": [
|
|
8822
|
+
"cento",
|
|
8823
|
+
"centinaio",
|
|
8824
|
+
"centinaia",
|
|
8825
|
+
"mille",
|
|
8826
|
+
"milione",
|
|
8827
|
+
"milioni",
|
|
8828
|
+
"miliardo",
|
|
8829
|
+
"miliardi",
|
|
8830
|
+
"bilione",
|
|
8831
|
+
"bilioni"
|
|
8832
|
+
],
|
|
8833
|
+
"abbreviationsCaseInsensitive": ["mln", "mld"],
|
|
8834
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8835
|
+
},
|
|
8836
|
+
{
|
|
8837
|
+
"lang": "pl",
|
|
8838
|
+
"words": [
|
|
8839
|
+
"sto",
|
|
8840
|
+
"sta",
|
|
8841
|
+
"stów",
|
|
8842
|
+
"stoma",
|
|
8843
|
+
"setki",
|
|
8844
|
+
"setek",
|
|
8845
|
+
"tysiąc",
|
|
8846
|
+
"tysiące",
|
|
8847
|
+
"tysięcy",
|
|
8848
|
+
"milion",
|
|
8849
|
+
"miliony",
|
|
8850
|
+
"milionów",
|
|
8851
|
+
"miliard",
|
|
8852
|
+
"miliardy",
|
|
8853
|
+
"miliardów",
|
|
8854
|
+
"bilion",
|
|
8855
|
+
"biliony",
|
|
8856
|
+
"bilionów"
|
|
8857
|
+
],
|
|
8858
|
+
"abbreviationsCaseInsensitive": ["mln", "mld"],
|
|
8859
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8860
|
+
},
|
|
8861
|
+
{
|
|
8862
|
+
"lang": "pt-br",
|
|
8863
|
+
"words": [
|
|
8864
|
+
"cem",
|
|
8865
|
+
"cento",
|
|
8866
|
+
"centena",
|
|
8867
|
+
"centenas",
|
|
8868
|
+
"milhão",
|
|
8869
|
+
"milhões",
|
|
8870
|
+
"bilhão",
|
|
8871
|
+
"bilhões",
|
|
8872
|
+
"trilhão",
|
|
8873
|
+
"trilhões"
|
|
8874
|
+
],
|
|
8875
|
+
"abbreviationsCaseInsensitive": [],
|
|
8876
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8877
|
+
},
|
|
8878
|
+
{
|
|
8879
|
+
"lang": "sk",
|
|
8880
|
+
"words": [
|
|
8881
|
+
"sto",
|
|
8882
|
+
"sta",
|
|
8883
|
+
"stoviek",
|
|
8884
|
+
"stovka",
|
|
8885
|
+
"stovky",
|
|
8886
|
+
"tisíc",
|
|
8887
|
+
"tisíce",
|
|
8888
|
+
"tisícov",
|
|
8889
|
+
"milión",
|
|
8890
|
+
"milióny",
|
|
8891
|
+
"miliónov",
|
|
8892
|
+
"milion",
|
|
8893
|
+
"miliony",
|
|
8894
|
+
"milionov",
|
|
8895
|
+
"miliarda",
|
|
8896
|
+
"miliardy",
|
|
8897
|
+
"miliárd",
|
|
8898
|
+
"bilión",
|
|
8899
|
+
"bilióny",
|
|
8900
|
+
"biliónov"
|
|
8901
|
+
],
|
|
8902
|
+
"abbreviationsCaseInsensitive": ["mld"],
|
|
8903
|
+
"abbreviationsCaseSensitive": ["K", "M"]
|
|
8904
|
+
}
|
|
8905
|
+
];
|
|
8906
|
+
var shareQuantityTerms = [
|
|
8907
|
+
{
|
|
8908
|
+
"lang": "cs",
|
|
8909
|
+
"modifiers": [
|
|
8910
|
+
"kmenové",
|
|
8911
|
+
"obyčejné",
|
|
8912
|
+
"prioritní",
|
|
8913
|
+
"registrované",
|
|
8914
|
+
"vlastní",
|
|
8915
|
+
"hlasovací",
|
|
8916
|
+
"nehlasovací",
|
|
8917
|
+
"omezené",
|
|
8918
|
+
"třída",
|
|
8919
|
+
"série"
|
|
8920
|
+
],
|
|
8921
|
+
"nouns": [
|
|
8922
|
+
"akcie",
|
|
8923
|
+
"cenné papíry",
|
|
8924
|
+
"podíly",
|
|
8925
|
+
"jednotky"
|
|
8926
|
+
]
|
|
8927
|
+
},
|
|
8928
|
+
{
|
|
8929
|
+
"lang": "de",
|
|
8930
|
+
"modifiers": [
|
|
8931
|
+
"stamm",
|
|
8932
|
+
"vorzugs",
|
|
8933
|
+
"namens",
|
|
8934
|
+
"eigene",
|
|
8935
|
+
"stimmberechtigte",
|
|
8936
|
+
"stimmrechtslose",
|
|
8937
|
+
"beschränkte",
|
|
8938
|
+
"klasse",
|
|
8939
|
+
"serie"
|
|
8940
|
+
],
|
|
8941
|
+
"nouns": [
|
|
8942
|
+
"aktien",
|
|
8943
|
+
"wertpapiere",
|
|
8944
|
+
"anteile",
|
|
8945
|
+
"einheiten"
|
|
8946
|
+
]
|
|
8947
|
+
},
|
|
8948
|
+
{
|
|
8949
|
+
"lang": "en",
|
|
8950
|
+
"modifiers": [
|
|
8951
|
+
"ordinary",
|
|
8952
|
+
"common",
|
|
8953
|
+
"preferred",
|
|
8954
|
+
"registered",
|
|
8955
|
+
"treasury",
|
|
8956
|
+
"voting",
|
|
8957
|
+
"non-voting",
|
|
8958
|
+
"restricted",
|
|
8959
|
+
"class",
|
|
8960
|
+
"series"
|
|
8961
|
+
],
|
|
8962
|
+
"nouns": [
|
|
8963
|
+
"share",
|
|
8964
|
+
"shares",
|
|
8965
|
+
"stock",
|
|
8966
|
+
"stocks",
|
|
8967
|
+
"security",
|
|
8968
|
+
"securities",
|
|
8969
|
+
"unit",
|
|
8970
|
+
"units"
|
|
8971
|
+
]
|
|
8972
|
+
},
|
|
8973
|
+
{
|
|
8974
|
+
"lang": "es",
|
|
8975
|
+
"modifiers": [
|
|
8976
|
+
"ordinarias",
|
|
8977
|
+
"comunes",
|
|
8978
|
+
"preferentes",
|
|
8979
|
+
"nominativas",
|
|
8980
|
+
"propias",
|
|
8981
|
+
"con derecho a voto",
|
|
8982
|
+
"sin derecho a voto",
|
|
8983
|
+
"restringidas",
|
|
8984
|
+
"clase",
|
|
8985
|
+
"serie"
|
|
8986
|
+
],
|
|
8987
|
+
"nouns": [
|
|
8988
|
+
"acciones",
|
|
8989
|
+
"valores",
|
|
8990
|
+
"participaciones",
|
|
8991
|
+
"unidades"
|
|
8992
|
+
]
|
|
8993
|
+
},
|
|
8994
|
+
{
|
|
8995
|
+
"lang": "fr",
|
|
8996
|
+
"modifiers": [
|
|
8997
|
+
"ordinaires",
|
|
8998
|
+
"communes",
|
|
8999
|
+
"privilégiées",
|
|
9000
|
+
"nominatives",
|
|
9001
|
+
"propres",
|
|
9002
|
+
"votantes",
|
|
9003
|
+
"sans droit de vote",
|
|
9004
|
+
"restreintes",
|
|
9005
|
+
"classe",
|
|
9006
|
+
"série"
|
|
9007
|
+
],
|
|
9008
|
+
"nouns": [
|
|
9009
|
+
"actions",
|
|
9010
|
+
"titres",
|
|
9011
|
+
"unités"
|
|
9012
|
+
]
|
|
9013
|
+
},
|
|
9014
|
+
{
|
|
9015
|
+
"lang": "it",
|
|
9016
|
+
"modifiers": [
|
|
9017
|
+
"ordinarie",
|
|
9018
|
+
"comuni",
|
|
9019
|
+
"privilegiate",
|
|
9020
|
+
"nominative",
|
|
9021
|
+
"proprie",
|
|
9022
|
+
"votanti",
|
|
9023
|
+
"senza diritto di voto",
|
|
9024
|
+
"vincolate",
|
|
9025
|
+
"classe",
|
|
9026
|
+
"serie"
|
|
9027
|
+
],
|
|
9028
|
+
"nouns": [
|
|
9029
|
+
"azioni",
|
|
9030
|
+
"titoli",
|
|
9031
|
+
"quote",
|
|
9032
|
+
"unità"
|
|
9033
|
+
]
|
|
9034
|
+
},
|
|
9035
|
+
{
|
|
9036
|
+
"lang": "pl",
|
|
9037
|
+
"modifiers": [
|
|
9038
|
+
"zwykłe",
|
|
9039
|
+
"imienne",
|
|
9040
|
+
"uprzywilejowane",
|
|
9041
|
+
"własne",
|
|
9042
|
+
"głosujące",
|
|
9043
|
+
"nieme",
|
|
9044
|
+
"ograniczone",
|
|
9045
|
+
"klasa",
|
|
9046
|
+
"seria"
|
|
9047
|
+
],
|
|
9048
|
+
"nouns": [
|
|
9049
|
+
"akcje",
|
|
9050
|
+
"papiery wartościowe",
|
|
9051
|
+
"udziały",
|
|
9052
|
+
"jednostki"
|
|
9053
|
+
]
|
|
9054
|
+
},
|
|
9055
|
+
{
|
|
9056
|
+
"lang": "pt-br",
|
|
9057
|
+
"modifiers": [
|
|
9058
|
+
"ordinárias",
|
|
9059
|
+
"comuns",
|
|
9060
|
+
"preferenciais",
|
|
9061
|
+
"nominativas",
|
|
9062
|
+
"próprias",
|
|
9063
|
+
"votantes",
|
|
9064
|
+
"sem direito a voto",
|
|
9065
|
+
"restritas",
|
|
9066
|
+
"classe",
|
|
9067
|
+
"série"
|
|
9068
|
+
],
|
|
9069
|
+
"nouns": [
|
|
9070
|
+
"ações",
|
|
9071
|
+
"valores mobiliários",
|
|
9072
|
+
"participações",
|
|
9073
|
+
"unidades"
|
|
9074
|
+
]
|
|
9075
|
+
},
|
|
9076
|
+
{
|
|
9077
|
+
"lang": "sk",
|
|
9078
|
+
"modifiers": [
|
|
9079
|
+
"kmeňové",
|
|
9080
|
+
"obyčajné",
|
|
9081
|
+
"prioritné",
|
|
9082
|
+
"registrované",
|
|
9083
|
+
"vlastné",
|
|
9084
|
+
"hlasovacie",
|
|
9085
|
+
"nehlasovacie",
|
|
9086
|
+
"obmedzené",
|
|
9087
|
+
"trieda",
|
|
9088
|
+
"séria"
|
|
9089
|
+
],
|
|
9090
|
+
"nouns": [
|
|
9091
|
+
"akcie",
|
|
9092
|
+
"cenné papiere",
|
|
9093
|
+
"podiely",
|
|
9094
|
+
"jednotky"
|
|
9095
|
+
]
|
|
9096
|
+
}
|
|
9097
|
+
];
|
|
9098
|
+
var amount_words_default = {
|
|
9099
|
+
_comment: _comment$1,
|
|
9100
|
+
patterns,
|
|
9101
|
+
percentages,
|
|
9102
|
+
magnitudeSuffixes,
|
|
9103
|
+
shareQuantityTerms
|
|
9104
|
+
};
|
|
9105
|
+
//#endregion
|
|
8640
9106
|
//#region src/detectors/regex.ts
|
|
8641
9107
|
const MIN_PHONE_LENGTH = 7;
|
|
8642
9108
|
const MIN_MONTH_NAME_LENGTH = 3;
|
|
@@ -8734,7 +9200,13 @@ const STDNUM_ENTRIES = [
|
|
|
8734
9200
|
toEntry(lu.vat, "tax identification number", .95),
|
|
8735
9201
|
toEntry(us.ein, "tax identification number", .9),
|
|
8736
9202
|
toEntry(br.cpf, "tax identification number", .95),
|
|
8737
|
-
toEntry(br.cnpj, "tax identification number", .95)
|
|
9203
|
+
toEntry(br.cnpj, "tax identification number", .95),
|
|
9204
|
+
{
|
|
9205
|
+
validator: cn.ric,
|
|
9206
|
+
label: "national identification number",
|
|
9207
|
+
score: .95,
|
|
9208
|
+
pattern: "(?<![A-Za-z0-9_])\\d{17}[\\dXx](?![A-Za-z0-9_])"
|
|
9209
|
+
}
|
|
8738
9210
|
].filter((e) => e !== null);
|
|
8739
9211
|
const TITLED_PERSON = {
|
|
8740
9212
|
pattern: `(?:${TITLE_PREFIX})(?:${SP}+(?:${TITLE_PREFIX}))*${SP}+(?:${NAME_WORD})(?:${SP}{1,4}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3}(?:,?${SP}+(?:${POST_NOMINAL})(?:,?${SP}+(?:${POST_NOMINAL}))*)?`,
|
|
@@ -8781,6 +9253,24 @@ const TEL_PREFIX_PHONE = {
|
|
|
8781
9253
|
label: "phone number",
|
|
8782
9254
|
score: .95
|
|
8783
9255
|
};
|
|
9256
|
+
/**
|
|
9257
|
+
* US phone numbers in the (NNN) NNN-NNNN form. Dominant
|
|
9258
|
+
* shape in US notice blocks ("(212) 735-3000"); not
|
|
9259
|
+
* covered by INTL_PHONE (which requires a leading `+`)
|
|
9260
|
+
* or TEL_PREFIX_PHONE (which requires a `tel.:` label).
|
|
9261
|
+
*
|
|
9262
|
+
* The parenthesised area code is the constraint that
|
|
9263
|
+
* keeps this from matching random digit clusters —
|
|
9264
|
+
* `(212) 555-1212` looks like nothing else in a contract.
|
|
9265
|
+
* Score below INTL_PHONE (1) and TEL_PREFIX_PHONE (0.95)
|
|
9266
|
+
* so labelled / fully-qualified forms still win the
|
|
9267
|
+
* overlap resolver when both fire on the same span.
|
|
9268
|
+
*/
|
|
9269
|
+
const US_PAREN_PHONE = {
|
|
9270
|
+
pattern: "\\(\\d{3}\\)(?:[^\\S\\n]|[.\\-])?\\d{3}(?:[^\\S\\n]|[.\\-])\\d{4}\\b",
|
|
9271
|
+
label: "phone number",
|
|
9272
|
+
score: .9
|
|
9273
|
+
};
|
|
8784
9274
|
const CREDIT_CARD = {
|
|
8785
9275
|
pattern: "\\b(?:4\\d{3}|5[1-5]\\d{2}|3[47]\\d{2})(?:[^\\S\\n]|[.\\-])?\\d{4}(?:[^\\S\\n]|[.\\-])?\\d{4}(?:[^\\S\\n]|[.\\-])?\\d{2,4}\\b",
|
|
8786
9276
|
label: "credit card number",
|
|
@@ -8982,6 +9472,7 @@ const ALL_REGEX_DEFS = [
|
|
|
8982
9472
|
INTL_PHONE,
|
|
8983
9473
|
CZ_PHONE,
|
|
8984
9474
|
TEL_PREFIX_PHONE,
|
|
9475
|
+
US_PAREN_PHONE,
|
|
8985
9476
|
CREDIT_CARD,
|
|
8986
9477
|
CZ_BIRTH_NUMBER,
|
|
8987
9478
|
CZ_COMMERCIAL_REGISTER,
|
|
@@ -9103,7 +9594,16 @@ const buildMagnitudePattern = (config) => {
|
|
|
9103
9594
|
if (wordsAlt) branches.push(`[^\\S\\n\\t]+(?i:(?:${wordsAlt}))\\b`);
|
|
9104
9595
|
if (abbreviationCiAlt) branches.push(`[^\\S\\n\\t]?(?i:${abbreviationCiAlt})\\b`);
|
|
9105
9596
|
if (abbreviationCsAlt) branches.push(`[^\\S\\n\\t]?(?:${abbreviationCsAlt})\\b`);
|
|
9106
|
-
|
|
9597
|
+
const required = branches.length > 0 ? `(?:${branches.join("|")})` : "";
|
|
9598
|
+
return {
|
|
9599
|
+
optional: required ? `${required}?` : "",
|
|
9600
|
+
required,
|
|
9601
|
+
prefilterTerms: [
|
|
9602
|
+
...words,
|
|
9603
|
+
...caseInsensitiveAbbreviations,
|
|
9604
|
+
...caseSensitiveAbbreviations
|
|
9605
|
+
]
|
|
9606
|
+
};
|
|
9107
9607
|
};
|
|
9108
9608
|
const buildQuantityFollowerGuard = (config) => {
|
|
9109
9609
|
const modifiers = [];
|
|
@@ -9117,10 +9617,15 @@ const buildQuantityFollowerGuard = (config) => {
|
|
|
9117
9617
|
if (!nounAlt) return "";
|
|
9118
9618
|
return `(?![^\\S\\n\\t]+(?i:${modifierAlt ? `(?:(?:${modifierAlt})[^\\S\\n\\t]+){0,3}` : ""}(?:${nounAlt}))\\b)`;
|
|
9119
9619
|
};
|
|
9120
|
-
const buildFinancialLexicons = (config) =>
|
|
9121
|
-
magnitude
|
|
9122
|
-
|
|
9123
|
-
|
|
9620
|
+
const buildFinancialLexicons = (config) => {
|
|
9621
|
+
const magnitude = buildMagnitudePattern(config);
|
|
9622
|
+
return Object.freeze({
|
|
9623
|
+
magnitudeOptional: magnitude.optional,
|
|
9624
|
+
magnitudeRequired: magnitude.required,
|
|
9625
|
+
magnitudePrefilterTerms: magnitude.prefilterTerms,
|
|
9626
|
+
quantityFollowerGuard: buildQuantityFollowerGuard(config)
|
|
9627
|
+
});
|
|
9628
|
+
};
|
|
9124
9629
|
const FINANCIAL_LEXICONS = buildFinancialLexicons(AMOUNT_WORDS);
|
|
9125
9630
|
/**
|
|
9126
9631
|
* Build symbol character class, code alternation,
|
|
@@ -9132,50 +9637,113 @@ const FINANCIAL_LEXICONS = buildFinancialLexicons(AMOUNT_WORDS);
|
|
|
9132
9637
|
* thousands (1,000) and plain integers (100000)
|
|
9133
9638
|
* via `\d{1,9}` to catch unformatted amounts.
|
|
9134
9639
|
*/
|
|
9135
|
-
const
|
|
9640
|
+
const buildCurrencyPatternEntries = (data) => {
|
|
9136
9641
|
const symbols = data.symbols.map(escapeCharClass).join("");
|
|
9137
9642
|
const isAsciiAlpha = /^[a-zA-Z\s]+$/;
|
|
9138
|
-
const
|
|
9643
|
+
const codeParts = data.codes.map((code) => ({
|
|
9644
|
+
term: code,
|
|
9139
9645
|
len: code.length,
|
|
9140
9646
|
alt: escapeRegex$2(code)
|
|
9141
9647
|
}));
|
|
9648
|
+
const localNameParts = [];
|
|
9142
9649
|
const MIN_CI_LENGTH = 3;
|
|
9143
9650
|
if (data.localNames) for (const name of data.localNames) {
|
|
9144
9651
|
const escaped = escapeRegex$2(name);
|
|
9145
|
-
if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH)
|
|
9652
|
+
if (isAsciiAlpha.test(name) && name.length >= MIN_CI_LENGTH) localNameParts.push({
|
|
9653
|
+
term: name,
|
|
9146
9654
|
len: name.length,
|
|
9147
9655
|
alt: `(?i:${escaped})`
|
|
9148
9656
|
});
|
|
9149
|
-
else
|
|
9657
|
+
else localNameParts.push({
|
|
9658
|
+
term: name,
|
|
9150
9659
|
len: name.length,
|
|
9151
9660
|
alt: escaped
|
|
9152
9661
|
});
|
|
9153
9662
|
}
|
|
9154
|
-
|
|
9155
|
-
|
|
9156
|
-
|
|
9157
|
-
|
|
9158
|
-
const
|
|
9663
|
+
const toPartAlternation = (parts) => parts.toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
|
|
9664
|
+
const codeAlt = toPartAlternation(codeParts);
|
|
9665
|
+
const localNameAlt = toPartAlternation(localNameParts);
|
|
9666
|
+
const codeTerms = codeParts.map((part) => part.term);
|
|
9667
|
+
const localNameTerms = localNameParts.map((part) => part.term);
|
|
9668
|
+
const trailingAlt = [...codeParts, ...localNameParts].toSorted((a, b) => b.len - a.len).map((p) => p.alt).join("|");
|
|
9159
9669
|
if (!symbols && !trailingAlt) return [];
|
|
9160
9670
|
const NUM = "(?:\\d{1,3}(?:[,.'[^\\S\\n\\t]]\\d{3})+|\\d{1,9})";
|
|
9671
|
+
const PREFILTER_NUM = `(?:\\d{1,3}(?:[,.'\\s]\\d{3})+|\\d{1,9})`;
|
|
9672
|
+
const PREFILTER_DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])\\s?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
|
|
9161
9673
|
const patterns = [];
|
|
9674
|
+
const lazyCurrencyPattern = (pattern, prefilterAny, prefilterCaseInsensitive, prefilterRegex) => ({
|
|
9675
|
+
pattern,
|
|
9676
|
+
lazy: true,
|
|
9677
|
+
prefilterAny,
|
|
9678
|
+
prefilterCaseInsensitive,
|
|
9679
|
+
...prefilterRegex ? { prefilterRegex } : {}
|
|
9680
|
+
});
|
|
9681
|
+
const makeLeadingPrefilter = (terms, caseInsensitive) => {
|
|
9682
|
+
const termAlt = toSortedAlternation(terms);
|
|
9683
|
+
if (!termAlt) return void 0;
|
|
9684
|
+
return new RegExp(`(?:^|[^\\p{L}\\p{N}_])(?:${termAlt})[^\\S\\n\\t]{0,2}\\d`, caseInsensitive ? "iu" : "u");
|
|
9685
|
+
};
|
|
9686
|
+
const makeTrailingPrefilter = (terms, caseInsensitive) => {
|
|
9687
|
+
const termAlt = toSortedAlternation(terms);
|
|
9688
|
+
if (!termAlt) return void 0;
|
|
9689
|
+
return new RegExp(`${PREFILTER_NUM}${PREFILTER_DECIMAL}[^\\S\\n\\t]{0,4}(?:${termAlt})`, caseInsensitive ? "iu" : "u");
|
|
9690
|
+
};
|
|
9691
|
+
const makeLeadingMagnitudePrefilter = (terms, caseInsensitive) => {
|
|
9692
|
+
const termAlt = toSortedAlternation(terms);
|
|
9693
|
+
const magnitudeAlt = toSortedAlternation(FINANCIAL_LEXICONS.magnitudePrefilterTerms);
|
|
9694
|
+
if (!termAlt || !magnitudeAlt) return void 0;
|
|
9695
|
+
return new RegExp(`(?:^|[^\\p{L}\\p{N}_])(?:${termAlt})[^\\S\\n\\t]{0,2}${PREFILTER_NUM}${PREFILTER_DECIMAL}[^\\S\\n\\t]{0,8}(?:${magnitudeAlt})(?:$|[^\\p{L}\\p{N}_])`, caseInsensitive ? "iu" : "u");
|
|
9696
|
+
};
|
|
9697
|
+
const makeLeadingSymbolMagnitudePrefilter = () => {
|
|
9698
|
+
const magnitudeAlt = toSortedAlternation(FINANCIAL_LEXICONS.magnitudePrefilterTerms);
|
|
9699
|
+
if (!magnitudeAlt) return void 0;
|
|
9700
|
+
return new RegExp(`(?:^|[^\\p{L}\\p{N}_])(?:[${symbols}])[^\\S\\n\\t]{0,2}${PREFILTER_NUM}${PREFILTER_DECIMAL}[^\\S\\n\\t]{0,8}(?:${magnitudeAlt})(?:$|[^\\p{L}\\p{N}_])`, "iu");
|
|
9701
|
+
};
|
|
9702
|
+
const makeTrailingMagnitudePrefilter = (terms, caseInsensitive) => {
|
|
9703
|
+
const termAlt = toSortedAlternation(terms);
|
|
9704
|
+
const magnitudeAlt = toSortedAlternation(FINANCIAL_LEXICONS.magnitudePrefilterTerms);
|
|
9705
|
+
if (!termAlt || !magnitudeAlt) return void 0;
|
|
9706
|
+
return new RegExp(`${PREFILTER_NUM}${PREFILTER_DECIMAL}[^\\S\\n\\t]{0,8}(?:${magnitudeAlt})(?:$|[^\\p{L}\\p{N}_])[\\s\\S]{0,24}(?:${termAlt})`, caseInsensitive ? "iu" : "u");
|
|
9707
|
+
};
|
|
9162
9708
|
const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
|
|
9163
9709
|
const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
|
|
9164
|
-
const
|
|
9165
|
-
|
|
9166
|
-
if (
|
|
9167
|
-
|
|
9168
|
-
|
|
9169
|
-
|
|
9710
|
+
const MAGNITUDE_OPTIONAL = FINANCIAL_LEXICONS.magnitudeOptional;
|
|
9711
|
+
const MAGNITUDE_REQUIRED = FINANCIAL_LEXICONS.magnitudeRequired;
|
|
9712
|
+
if (symbols) {
|
|
9713
|
+
patterns.push(lazyCurrencyPattern(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${END}`, data.symbols, true));
|
|
9714
|
+
if (MAGNITUDE_REQUIRED) patterns.push(lazyCurrencyPattern(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${MAGNITUDE_REQUIRED}${END}`, data.symbols, true, makeLeadingSymbolMagnitudePrefilter()));
|
|
9715
|
+
}
|
|
9716
|
+
if (codeAlt) {
|
|
9717
|
+
patterns.push(lazyCurrencyPattern(`\\b(?:${codeAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${END}`, codeTerms, false, makeLeadingPrefilter(codeTerms, false)));
|
|
9718
|
+
if (MAGNITUDE_REQUIRED) patterns.push(lazyCurrencyPattern(`\\b(?:${codeAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${MAGNITUDE_REQUIRED}${END}`, codeTerms, false, makeLeadingMagnitudePrefilter(codeTerms, false)));
|
|
9719
|
+
}
|
|
9720
|
+
if (localNameAlt) {
|
|
9721
|
+
patterns.push(lazyCurrencyPattern(`\\b(?:${localNameAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${END}`, localNameTerms, true, makeLeadingPrefilter(localNameTerms, true)));
|
|
9722
|
+
if (MAGNITUDE_REQUIRED) patterns.push(lazyCurrencyPattern(`\\b(?:${localNameAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${MAGNITUDE_REQUIRED}${END}`, localNameTerms, true, makeLeadingMagnitudePrefilter(localNameTerms, true)));
|
|
9723
|
+
}
|
|
9724
|
+
const optionalLeadingSymbol = symbols ? `(?<![\\p{L}\\p{N}_])(?:[${symbols}][^\\S\\n\\t]?)?` : "\\b";
|
|
9725
|
+
if (codeAlt) {
|
|
9726
|
+
patterns.push(lazyCurrencyPattern(`${optionalLeadingSymbol}${NUM}${DECIMAL}[^\\S\\n\\t]{0,4}(?:${codeAlt})${END}`, codeTerms, false, makeTrailingPrefilter(codeTerms, false)));
|
|
9727
|
+
if (MAGNITUDE_REQUIRED) patterns.push(lazyCurrencyPattern(`${optionalLeadingSymbol}${NUM}${DECIMAL}${MAGNITUDE_REQUIRED}[^\\S\\n\\t]{0,4}(?:${codeAlt})${FINANCIAL_LEXICONS.quantityFollowerGuard}${END}`, codeTerms, false, makeTrailingMagnitudePrefilter(codeTerms, false)));
|
|
9728
|
+
}
|
|
9729
|
+
if (localNameAlt) {
|
|
9730
|
+
patterns.push(lazyCurrencyPattern(`${optionalLeadingSymbol}${NUM}${DECIMAL}[^\\S\\n\\t]{0,4}(?:${localNameAlt})${END}`, localNameTerms, true, makeTrailingPrefilter(localNameTerms, true)));
|
|
9731
|
+
if (MAGNITUDE_REQUIRED) patterns.push(lazyCurrencyPattern(`${optionalLeadingSymbol}${NUM}${DECIMAL}${MAGNITUDE_REQUIRED}[^\\S\\n\\t]{0,4}(?:${localNameAlt})${FINANCIAL_LEXICONS.quantityFollowerGuard}${END}`, localNameTerms, true, makeTrailingMagnitudePrefilter(localNameTerms, true)));
|
|
9732
|
+
}
|
|
9733
|
+
if (symbols) {
|
|
9734
|
+
const trailingSymbolPrefilter = new RegExp(`${PREFILTER_NUM}${PREFILTER_DECIMAL}[^\\S\\n\\t]{0,4}[${symbols}]`, "u");
|
|
9735
|
+
patterns.push(lazyCurrencyPattern(`${NUM}${DECIMAL}${MAGNITUDE_OPTIONAL}[^\\S\\n\\t]{0,4}(?:[${symbols}])${END}`, data.symbols, true, trailingSymbolPrefilter));
|
|
9170
9736
|
}
|
|
9171
9737
|
return patterns;
|
|
9172
9738
|
};
|
|
9173
9739
|
/** Cached promise for currency patterns. Loaded once. */
|
|
9174
9740
|
let currencyPatternPromise = null;
|
|
9175
|
-
|
|
9741
|
+
let currencyPatternEntryPromise = null;
|
|
9742
|
+
const loadCurrencyPatternEntries = async () => {
|
|
9176
9743
|
const mod = await import("./currencies.mjs");
|
|
9177
|
-
return
|
|
9744
|
+
return buildCurrencyPatternEntries(mod.default ?? mod);
|
|
9178
9745
|
};
|
|
9746
|
+
const loadCurrencyPatterns = async () => (await loadCurrencyPatternEntries()).map((entry) => entry.pattern);
|
|
9179
9747
|
/**
|
|
9180
9748
|
* Get dynamically built monetary amount patterns from
|
|
9181
9749
|
* currencies.json. Returns a cached promise; the JSON
|
|
@@ -9188,6 +9756,13 @@ const getCurrencyPatterns = () => {
|
|
|
9188
9756
|
});
|
|
9189
9757
|
return currencyPatternPromise;
|
|
9190
9758
|
};
|
|
9759
|
+
const getCurrencyPatternEntries = () => {
|
|
9760
|
+
if (!currencyPatternEntryPromise) currencyPatternEntryPromise = loadCurrencyPatternEntries().catch((err) => {
|
|
9761
|
+
currencyPatternEntryPromise = null;
|
|
9762
|
+
throw err;
|
|
9763
|
+
});
|
|
9764
|
+
return currencyPatternEntryPromise;
|
|
9765
|
+
};
|
|
9191
9766
|
/** Currency pattern metadata (score 0.9). */
|
|
9192
9767
|
const CURRENCY_PATTERN_META = Object.freeze({
|
|
9193
9768
|
label: "monetary amount",
|
|
@@ -9268,6 +9843,249 @@ const getSigningClausePatterns = () => {
|
|
|
9268
9843
|
return signingPatternPromise;
|
|
9269
9844
|
};
|
|
9270
9845
|
//#endregion
|
|
9846
|
+
//#region src/detectors/legal-forms-v2.ts
|
|
9847
|
+
const normalizeSuffixToken = (s) => s.replace(/[.,\s]/g, "").toLowerCase();
|
|
9848
|
+
let cachedNormalizedSuffixes = null;
|
|
9849
|
+
const getNormalizedSuffixSet = () => {
|
|
9850
|
+
if (cachedNormalizedSuffixes !== null) return cachedNormalizedSuffixes;
|
|
9851
|
+
const out = /* @__PURE__ */ new Set();
|
|
9852
|
+
for (const s of getKnownLegalSuffixes()) {
|
|
9853
|
+
const n = normalizeSuffixToken(s);
|
|
9854
|
+
if (n.length > 0) out.add(n);
|
|
9855
|
+
}
|
|
9856
|
+
cachedNormalizedSuffixes = out;
|
|
9857
|
+
return out;
|
|
9858
|
+
};
|
|
9859
|
+
const isLegalFormSuffixWord = (word) => {
|
|
9860
|
+
const n = normalizeSuffixToken(word);
|
|
9861
|
+
if (n.length === 0) return false;
|
|
9862
|
+
return getNormalizedSuffixSet().has(n);
|
|
9863
|
+
};
|
|
9864
|
+
let cachedSuffixSearch = null;
|
|
9865
|
+
const getSuffixSearch = () => {
|
|
9866
|
+
const suffixes = getKnownLegalSuffixes();
|
|
9867
|
+
if (cachedSuffixSearch !== null && cachedSuffixSearch.suffixes === suffixes) return cachedSuffixSearch;
|
|
9868
|
+
const patterns = suffixes.map((suffix) => ({
|
|
9869
|
+
pattern: suffix,
|
|
9870
|
+
literal: true
|
|
9871
|
+
}));
|
|
9872
|
+
cachedSuffixSearch = {
|
|
9873
|
+
ts: new (getTextSearch())(patterns),
|
|
9874
|
+
suffixes
|
|
9875
|
+
};
|
|
9876
|
+
return cachedSuffixSearch;
|
|
9877
|
+
};
|
|
9878
|
+
const ANY_LETTER_RE = /\p{L}/u;
|
|
9879
|
+
const ANY_LETTER_OR_DIGIT_RE = /[\p{L}\p{N}]/u;
|
|
9880
|
+
const isTrailingBoundary = (fullText, end) => {
|
|
9881
|
+
if (end >= fullText.length) return true;
|
|
9882
|
+
const next = fullText.charAt(end);
|
|
9883
|
+
if (ANY_LETTER_RE.test(next)) return false;
|
|
9884
|
+
if (/\d/.test(next)) return false;
|
|
9885
|
+
return true;
|
|
9886
|
+
};
|
|
9887
|
+
const isLeadingSeparator = (fullText, suffixStart) => {
|
|
9888
|
+
if (suffixStart === 0) return true;
|
|
9889
|
+
const prev = fullText.charAt(suffixStart - 1);
|
|
9890
|
+
if (ANY_LETTER_OR_DIGIT_RE.test(prev)) return false;
|
|
9891
|
+
if (prev === "." && suffixStart >= 2 && ANY_LETTER_RE.test(fullText.charAt(suffixStart - 2))) return false;
|
|
9892
|
+
return true;
|
|
9893
|
+
};
|
|
9894
|
+
const HEAD_TOKEN_CAP = 20;
|
|
9895
|
+
const TOKEN_RE = /[\p{L}\p{N}'’.&-]+/u;
|
|
9896
|
+
const isInterTokenWs = (ch) => ch === " " || ch === " " || ch === "\xA0" || ch === " " || ch === ",";
|
|
9897
|
+
const findTokenBefore = (fullText, pos) => {
|
|
9898
|
+
let end = pos;
|
|
9899
|
+
while (end > 0) {
|
|
9900
|
+
const ch = fullText.charAt(end - 1);
|
|
9901
|
+
if (ch === "\n") return null;
|
|
9902
|
+
if (isInterTokenWs(ch) || ch === ";") {
|
|
9903
|
+
end--;
|
|
9904
|
+
continue;
|
|
9905
|
+
}
|
|
9906
|
+
break;
|
|
9907
|
+
}
|
|
9908
|
+
if (end === 0) return null;
|
|
9909
|
+
let start = end;
|
|
9910
|
+
while (start > 0) {
|
|
9911
|
+
const ch = fullText.charAt(start - 1);
|
|
9912
|
+
if (ch === "\n") break;
|
|
9913
|
+
if (!TOKEN_RE.test(ch)) break;
|
|
9914
|
+
start--;
|
|
9915
|
+
}
|
|
9916
|
+
if (start === end) return null;
|
|
9917
|
+
return {
|
|
9918
|
+
start,
|
|
9919
|
+
end,
|
|
9920
|
+
text: fullText.slice(start, end)
|
|
9921
|
+
};
|
|
9922
|
+
};
|
|
9923
|
+
const crossesSentenceEnd = (fullText, candidateStart, suffixStart) => {
|
|
9924
|
+
const slice = fullText.slice(candidateStart, suffixStart);
|
|
9925
|
+
return /\p{Lu}\p{Ll}{2,}\.\s/u.test(slice) || /\p{Lu}{2,}\.\s/u.test(slice);
|
|
9926
|
+
};
|
|
9927
|
+
const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
9928
|
+
const LOWER_LETTER_RE = /^\p{Ll}/u;
|
|
9929
|
+
const DIGIT_RE = /^\d/;
|
|
9930
|
+
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
9931
|
+
const isAcceptableToken = (tok) => {
|
|
9932
|
+
if (tok.length === 0) return false;
|
|
9933
|
+
if (UPPER_LETTER_RE.test(tok)) return true;
|
|
9934
|
+
if (DIGIT_RE.test(tok)) return true;
|
|
9935
|
+
if (CONNECTOR_RE.test(tok)) return true;
|
|
9936
|
+
if (LOWER_LETTER_RE.test(tok)) return true;
|
|
9937
|
+
return false;
|
|
9938
|
+
};
|
|
9939
|
+
const AND_TYPE_CONNECTOR_RE = /^(?:and|und|et)$/i;
|
|
9940
|
+
const MIDDLE_INITIAL_RE = /\p{Lu}\.[^\S\n]+\p{L}[\p{L}\p{M}'’]*[^\S\n]*$/u;
|
|
9941
|
+
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
9942
|
+
const slice = fullText.slice(Math.max(0, pos - 32), pos);
|
|
9943
|
+
return MIDDLE_INITIAL_RE.test(slice);
|
|
9944
|
+
};
|
|
9945
|
+
/**
|
|
9946
|
+
* Count consecutive uppercase-starting tokens immediately before
|
|
9947
|
+
* `pos`. Stops at the first non-upper token, a hard newline, or
|
|
9948
|
+
* text start.
|
|
9949
|
+
*/
|
|
9950
|
+
const countUpperBefore = (fullText, pos) => {
|
|
9951
|
+
let scan = pos;
|
|
9952
|
+
let count = 0;
|
|
9953
|
+
while (true) {
|
|
9954
|
+
const tok = findTokenBefore(fullText, scan);
|
|
9955
|
+
if (!tok) break;
|
|
9956
|
+
if (!UPPER_LETTER_RE.test(tok.text)) break;
|
|
9957
|
+
count++;
|
|
9958
|
+
scan = tok.start;
|
|
9959
|
+
}
|
|
9960
|
+
return count;
|
|
9961
|
+
};
|
|
9962
|
+
const MAX_LOWER_BRIDGE = 4;
|
|
9963
|
+
const walkBackward = (fullText, suffixStart) => {
|
|
9964
|
+
let pos = suffixStart;
|
|
9965
|
+
let stepsLeft = HEAD_TOKEN_CAP;
|
|
9966
|
+
let leftmostCapPos = -1;
|
|
9967
|
+
let lowerBridgeRun = 0;
|
|
9968
|
+
while (stepsLeft > 0) {
|
|
9969
|
+
const tok = findTokenBefore(fullText, pos);
|
|
9970
|
+
if (!tok) break;
|
|
9971
|
+
if (!isAcceptableToken(tok.text)) break;
|
|
9972
|
+
if (LOWER_LETTER_RE.test(tok.text) && leftmostCapPos >= 0) {
|
|
9973
|
+
const afterTok = fullText.slice(tok.end, pos);
|
|
9974
|
+
if (/^[,;]/.test(afterTok) && isLegalFormSuffixWord(tok.text)) break;
|
|
9975
|
+
}
|
|
9976
|
+
if (CONNECTOR_RE.test(tok.text)) {
|
|
9977
|
+
const prevPeek = findTokenBefore(fullText, tok.start);
|
|
9978
|
+
if (prevPeek && isLegalFormSuffixWord(prevPeek.text)) break;
|
|
9979
|
+
if (AND_TYPE_CONNECTOR_RE.test(tok.text)) {
|
|
9980
|
+
if (countUpperBefore(fullText, tok.start) <= 2) break;
|
|
9981
|
+
if (hasMiddleInitialBefore(fullText, tok.start)) break;
|
|
9982
|
+
}
|
|
9983
|
+
}
|
|
9984
|
+
if (UPPER_LETTER_RE.test(tok.text)) {
|
|
9985
|
+
leftmostCapPos = tok.start;
|
|
9986
|
+
lowerBridgeRun = 0;
|
|
9987
|
+
} else if (LOWER_LETTER_RE.test(tok.text)) {
|
|
9988
|
+
if (leftmostCapPos >= 0) {
|
|
9989
|
+
lowerBridgeRun++;
|
|
9990
|
+
if (lowerBridgeRun > MAX_LOWER_BRIDGE) break;
|
|
9991
|
+
}
|
|
9992
|
+
} else lowerBridgeRun = 0;
|
|
9993
|
+
pos = tok.start;
|
|
9994
|
+
stepsLeft--;
|
|
9995
|
+
}
|
|
9996
|
+
return leftmostCapPos < 0 ? suffixStart : leftmostCapPos;
|
|
9997
|
+
};
|
|
9998
|
+
const CAP_TOKEN_RE = /(?<![\p{L}\p{N}])\p{Lu}[\p{L}\p{M}\p{N}'’.&-]*/gu;
|
|
9999
|
+
const LOWER_WORD_RE = /(?<![\p{L}\p{N}])\p{Ll}[\p{L}\p{M}'’]*/gu;
|
|
10000
|
+
/**
|
|
10001
|
+
* Post-walker verb gate. The walker admits any lowercase token so
|
|
10002
|
+
* Czech and German extended state-form tails (`Krajská správa,
|
|
10003
|
+
* příspěvková organizace`, `Národní agentura pro komunikační a
|
|
10004
|
+
* informační technologie, s. p.`) keep their full vocabulary. The
|
|
10005
|
+
* trade-off is that prose sentences ending in a legal-form descriptor
|
|
10006
|
+
* (`...převádí na příspěvková organizace.`) get swept too. When the
|
|
10007
|
+
* candidate text contains a known lowercase sentence-verb token we
|
|
10008
|
+
* slide `candidateStart` forward to the first capitalised token after
|
|
10009
|
+
* the last verb; if no capitalised token follows the verb, the
|
|
10010
|
+
* candidate is prose and gets dropped.
|
|
10011
|
+
*/
|
|
10012
|
+
const trimToFirstCapAfterVerb = (fullText, candidateStart, suffixStart) => {
|
|
10013
|
+
if (candidateStart >= suffixStart) return candidateStart;
|
|
10014
|
+
const head = fullText.slice(candidateStart, suffixStart);
|
|
10015
|
+
const verbIndicators = getSentenceVerbIndicatorsSync();
|
|
10016
|
+
let lastVerbEnd = -1;
|
|
10017
|
+
for (const wordMatch of head.matchAll(LOWER_WORD_RE)) if (wordMatch.index !== void 0 && verbIndicators.has(wordMatch[0].toLowerCase())) lastVerbEnd = wordMatch.index + wordMatch[0].length;
|
|
10018
|
+
if (lastVerbEnd < 0) return candidateStart;
|
|
10019
|
+
const roleHeads = getLegalRoleHeadsSync();
|
|
10020
|
+
const clauseNouns = getClauseNounHeadsSync();
|
|
10021
|
+
CAP_TOKEN_RE.lastIndex = lastVerbEnd;
|
|
10022
|
+
for (let next = CAP_TOKEN_RE.exec(head); next !== null; next = CAP_TOKEN_RE.exec(head)) {
|
|
10023
|
+
const word = next[0].toLowerCase();
|
|
10024
|
+
if (roleHeads.has(word) || clauseNouns.has(word)) continue;
|
|
10025
|
+
return candidateStart + next.index;
|
|
10026
|
+
}
|
|
10027
|
+
return suffixStart;
|
|
10028
|
+
};
|
|
10029
|
+
const synthMatch = (start, end, fullText) => ({
|
|
10030
|
+
start,
|
|
10031
|
+
end,
|
|
10032
|
+
pattern: 0,
|
|
10033
|
+
text: fullText.slice(start, end)
|
|
10034
|
+
});
|
|
10035
|
+
const detectLegalFormsV2 = (fullText) => {
|
|
10036
|
+
const { ts } = getSuffixSearch();
|
|
10037
|
+
const searchText = normalizeForSearch(fullText);
|
|
10038
|
+
const candidates = [];
|
|
10039
|
+
const trimmedCandidates = /* @__PURE__ */ new Set();
|
|
10040
|
+
for (const match of ts.findIter(searchText)) {
|
|
10041
|
+
const suffixStart = match.start;
|
|
10042
|
+
const suffixEnd = match.end;
|
|
10043
|
+
let effectiveSuffixStart = suffixStart;
|
|
10044
|
+
{
|
|
10045
|
+
let scan = suffixStart;
|
|
10046
|
+
while (scan > 0) {
|
|
10047
|
+
const ch = fullText.charAt(scan - 1);
|
|
10048
|
+
if (ch === " " || ch === " ") {
|
|
10049
|
+
scan--;
|
|
10050
|
+
continue;
|
|
10051
|
+
}
|
|
10052
|
+
break;
|
|
10053
|
+
}
|
|
10054
|
+
if (scan > 0 && fullText.charAt(scan - 1) === "\n") {
|
|
10055
|
+
let p = scan - 1;
|
|
10056
|
+
while (p > 0 && fullText.charAt(p - 1) === " ") p--;
|
|
10057
|
+
if (p > 0 && fullText.charAt(p - 1) === ".") effectiveSuffixStart = p;
|
|
10058
|
+
}
|
|
10059
|
+
}
|
|
10060
|
+
if (!isLeadingSeparator(fullText, suffixStart)) continue;
|
|
10061
|
+
if (!isTrailingBoundary(fullText, suffixEnd)) continue;
|
|
10062
|
+
const walkerStart = walkBackward(fullText, effectiveSuffixStart);
|
|
10063
|
+
if (walkerStart >= effectiveSuffixStart) continue;
|
|
10064
|
+
if (crossesSentenceEnd(fullText, walkerStart, effectiveSuffixStart)) continue;
|
|
10065
|
+
const candidateStart = trimToFirstCapAfterVerb(fullText, walkerStart, effectiveSuffixStart);
|
|
10066
|
+
if (candidateStart >= effectiveSuffixStart) continue;
|
|
10067
|
+
const candidate = synthMatch(candidateStart, suffixEnd, fullText);
|
|
10068
|
+
if (candidateStart !== walkerStart) trimmedCandidates.add(candidate);
|
|
10069
|
+
candidates.push(candidate);
|
|
10070
|
+
}
|
|
10071
|
+
if (candidates.length === 0) return [];
|
|
10072
|
+
const dedupedCandidates = dropOverlapping(candidates);
|
|
10073
|
+
const trimmed = [];
|
|
10074
|
+
const untrimmed = [];
|
|
10075
|
+
for (const c of dedupedCandidates) (trimmedCandidates.has(c) ? trimmed : untrimmed).push(c);
|
|
10076
|
+
return [...processLegalFormMatches(trimmed, 0, 1, fullText, { suppressExtendBackward: true }), ...processLegalFormMatches(untrimmed, 0, 1, fullText)];
|
|
10077
|
+
};
|
|
10078
|
+
const dropOverlapping = (candidates) => {
|
|
10079
|
+
const sorted = [...candidates].sort((a, b) => a.start - b.start || b.end - a.end);
|
|
10080
|
+
const out = [];
|
|
10081
|
+
for (const c of sorted) {
|
|
10082
|
+
const last = out.at(-1);
|
|
10083
|
+
if (last && c.start >= last.start && c.end <= last.end) continue;
|
|
10084
|
+
out.push(c);
|
|
10085
|
+
}
|
|
10086
|
+
return out;
|
|
10087
|
+
};
|
|
10088
|
+
//#endregion
|
|
9271
10089
|
//#region src/detectors/triggers.ts
|
|
9272
10090
|
const VALID_ID_VALIDATORS = {
|
|
9273
10091
|
"br.cpf": br.cpf,
|
|
@@ -9287,22 +10105,31 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
|
|
|
9287
10105
|
* etc.), skip the comma and degree, then continue.
|
|
9288
10106
|
*/
|
|
9289
10107
|
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
9290
|
-
const buildLegalFormCheckRe = (forms) => {
|
|
9291
|
-
const parts = forms.map((f) => {
|
|
9292
|
-
const escaped = f.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*");
|
|
9293
|
-
return /^[\p{L}\p{M}]+$/u.test(f) ? `(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])` : escaped;
|
|
9294
|
-
});
|
|
9295
|
-
return new RegExp(parts.join("|"), "u");
|
|
9296
|
-
};
|
|
9297
|
-
let cachedLegalFormCheckRe = null;
|
|
9298
10108
|
let cachedLegalFormCheckSource = null;
|
|
9299
|
-
|
|
10109
|
+
let cachedLegalFormCheckForms = [];
|
|
10110
|
+
const LEGAL_FORM_ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
10111
|
+
const LETTER_ONLY_LEGAL_FORM_RE = /^[\p{L}\p{M}]+$/u;
|
|
10112
|
+
const getLegalFormCheckForms = () => {
|
|
9300
10113
|
const source = getKnownLegalSuffixes();
|
|
9301
|
-
if (cachedLegalFormCheckSource !== source
|
|
10114
|
+
if (cachedLegalFormCheckSource !== source) {
|
|
9302
10115
|
cachedLegalFormCheckSource = source;
|
|
9303
|
-
|
|
10116
|
+
cachedLegalFormCheckForms = source;
|
|
10117
|
+
}
|
|
10118
|
+
return cachedLegalFormCheckForms;
|
|
10119
|
+
};
|
|
10120
|
+
const hasKnownLegalFormSuffix = (text) => {
|
|
10121
|
+
for (const form of getLegalFormCheckForms()) {
|
|
10122
|
+
let fromIndex = 0;
|
|
10123
|
+
while (fromIndex < text.length) {
|
|
10124
|
+
const start = text.indexOf(form, fromIndex);
|
|
10125
|
+
if (start === -1) break;
|
|
10126
|
+
const end = start + form.length;
|
|
10127
|
+
fromIndex = start + 1;
|
|
10128
|
+
if (!LETTER_ONLY_LEGAL_FORM_RE.test(form)) return true;
|
|
10129
|
+
if (!LEGAL_FORM_ALNUM_RE.test(text[start - 1] ?? "") && !LEGAL_FORM_ALNUM_RE.test(text[end] ?? "")) return true;
|
|
10130
|
+
}
|
|
9304
10131
|
}
|
|
9305
|
-
return
|
|
10132
|
+
return false;
|
|
9306
10133
|
};
|
|
9307
10134
|
const compileValidations = (validations) => validations.map((v) => {
|
|
9308
10135
|
switch (v.type) {
|
|
@@ -9396,12 +10223,13 @@ const expandTriggerGroups = (groups) => {
|
|
|
9396
10223
|
}
|
|
9397
10224
|
return rules;
|
|
9398
10225
|
};
|
|
10226
|
+
let triggerPatternsPromise = null;
|
|
9399
10227
|
/**
|
|
9400
10228
|
* Build trigger patterns and rules from data configs.
|
|
9401
10229
|
* Returns string[] for the unified TextSearch
|
|
9402
10230
|
* builder and the parallel rules array.
|
|
9403
10231
|
*/
|
|
9404
|
-
const
|
|
10232
|
+
const loadTriggerPatterns = async () => {
|
|
9405
10233
|
const rules = [];
|
|
9406
10234
|
const allGroups = await loadLanguageConfigs("triggers", (mod) => {
|
|
9407
10235
|
return mod.default ?? mod;
|
|
@@ -9468,6 +10296,10 @@ const buildTriggerPatterns = async () => {
|
|
|
9468
10296
|
rules
|
|
9469
10297
|
};
|
|
9470
10298
|
};
|
|
10299
|
+
const buildTriggerPatterns = async () => {
|
|
10300
|
+
triggerPatternsPromise ??= loadTriggerPatterns();
|
|
10301
|
+
return triggerPatternsPromise;
|
|
10302
|
+
};
|
|
9471
10303
|
const LEADING_PUNCT = /^[„""»«'"()\s]+/;
|
|
9472
10304
|
const TRAILING_PUNCT = /[""»«'"()\s]+$/;
|
|
9473
10305
|
const stripQuotes = (value) => {
|
|
@@ -9661,6 +10493,9 @@ const warmAddressStopKeywords = async () => {
|
|
|
9661
10493
|
};
|
|
9662
10494
|
const MAX_TRIGGER_VALUE_LEN = 100;
|
|
9663
10495
|
const MIN_TRIGGER_PHONE_DIGITS = 5;
|
|
10496
|
+
const TRIGGER_LOOKAHEAD_MARGIN = 128;
|
|
10497
|
+
const LINE_TRIGGER_LOOKAHEAD = 2048;
|
|
10498
|
+
const MATCH_PATTERN_LOOKAHEAD = 512;
|
|
9664
10499
|
const PHONE_VALUE_START_RE = /^[+(\d]/;
|
|
9665
10500
|
const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
|
|
9666
10501
|
const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
@@ -9680,8 +10515,20 @@ const isPlausiblePhoneTriggerValue = (value) => {
|
|
|
9680
10515
|
for (const ch of trimmed) if (/\d/.test(ch)) digits++;
|
|
9681
10516
|
return digits >= MIN_TRIGGER_PHONE_DIGITS;
|
|
9682
10517
|
};
|
|
10518
|
+
const getTriggerLookahead = (strategy) => {
|
|
10519
|
+
switch (strategy.type) {
|
|
10520
|
+
case "to-next-comma": return (strategy.maxLength ?? 100) + TRIGGER_LOOKAHEAD_MARGIN;
|
|
10521
|
+
case "to-end-of-line": return LINE_TRIGGER_LOOKAHEAD;
|
|
10522
|
+
case "n-words": return strategy.count * 64 + TRIGGER_LOOKAHEAD_MARGIN;
|
|
10523
|
+
case "company-id-value": return 256;
|
|
10524
|
+
case "address": return (strategy.maxChars ?? 120) + TRIGGER_LOOKAHEAD_MARGIN;
|
|
10525
|
+
case "match-pattern": return MATCH_PATTERN_LOOKAHEAD;
|
|
10526
|
+
default: throw new Error(`Unknown trigger strategy: ${JSON.stringify(strategy)}`);
|
|
10527
|
+
}
|
|
10528
|
+
};
|
|
9683
10529
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
9684
|
-
const
|
|
10530
|
+
const lookaheadEnd = Math.min(text.length, triggerEnd + getTriggerLookahead(strategy));
|
|
10531
|
+
const remaining = text.slice(triggerEnd, lookaheadEnd);
|
|
9685
10532
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
9686
10533
|
const valueStart = triggerEnd + (remaining.length - stripped.length);
|
|
9687
10534
|
const valueText = stripped;
|
|
@@ -9927,7 +10774,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
9927
10774
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
9928
10775
|
const entityEnd = value.end;
|
|
9929
10776
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
9930
|
-
const effectiveLabel = rule.label === "person" &&
|
|
10777
|
+
const effectiveLabel = rule.label === "person" && hasKnownLegalFormSuffix(entityText) ? "organization" : rule.label;
|
|
9931
10778
|
results.push({
|
|
9932
10779
|
start: entityStart,
|
|
9933
10780
|
end: entityEnd,
|
|
@@ -10293,7 +11140,7 @@ const ALL_CAPS_LINE_PROSE_EXTRA_LETTERS = 20;
|
|
|
10293
11140
|
const ALL_CAPS_LINE_HEADING_WORD_LIMIT = 5;
|
|
10294
11141
|
const SECTION_HEADING_PREFIX_RE = /^\s*(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?\s*\p{Lu}/u;
|
|
10295
11142
|
const LINE_LETTER_RE = /\p{L}/gu;
|
|
10296
|
-
const ENTITY_WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'
|
|
11143
|
+
const ENTITY_WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’-]*/gu;
|
|
10297
11144
|
const isAllCapsBoilerplateLine = (fullText, start, length) => {
|
|
10298
11145
|
const lineStart = fullText.lastIndexOf("\n", start) + 1;
|
|
10299
11146
|
const lineEndIdx = fullText.indexOf("\n", start + length);
|
|
@@ -10371,6 +11218,47 @@ const normalizeEntity = (entity) => {
|
|
|
10371
11218
|
text
|
|
10372
11219
|
};
|
|
10373
11220
|
};
|
|
11221
|
+
let cachedHeadingRe = null;
|
|
11222
|
+
let cachedHeadingPromise = null;
|
|
11223
|
+
const ORDINAL_MARKER = "(?:č|no|nr|n)\\.?";
|
|
11224
|
+
const buildHeadingRegex = (words) => {
|
|
11225
|
+
if (words.length === 0) return /[\s\S](?!)/u;
|
|
11226
|
+
const escaped = [...words].sort((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
|
11227
|
+
return new RegExp(`^(?:${escaped})[\\s\\u00a0]+(?:${ORDINAL_MARKER}|#)[\\s\\u00a0]*\\d`, "iu");
|
|
11228
|
+
};
|
|
11229
|
+
const loadHeadingWords = async () => {
|
|
11230
|
+
try {
|
|
11231
|
+
const mod = await import("./document-structure-headings.mjs");
|
|
11232
|
+
const data = mod.default ?? mod;
|
|
11233
|
+
const entries = Object.entries(data);
|
|
11234
|
+
const out = [];
|
|
11235
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11236
|
+
for (const [key, value] of entries) {
|
|
11237
|
+
if (key.startsWith("_")) continue;
|
|
11238
|
+
if (!Array.isArray(value)) continue;
|
|
11239
|
+
for (const word of value) {
|
|
11240
|
+
if (typeof word !== "string" || word.length === 0) continue;
|
|
11241
|
+
const lc = word.toLowerCase();
|
|
11242
|
+
if (seen.has(lc)) continue;
|
|
11243
|
+
seen.add(lc);
|
|
11244
|
+
out.push(lc);
|
|
11245
|
+
}
|
|
11246
|
+
}
|
|
11247
|
+
return out;
|
|
11248
|
+
} catch {
|
|
11249
|
+
return [];
|
|
11250
|
+
}
|
|
11251
|
+
};
|
|
11252
|
+
const loadDocumentStructureHeadings = async () => {
|
|
11253
|
+
if (cachedHeadingRe) return;
|
|
11254
|
+
cachedHeadingPromise ??= loadHeadingWords().then(buildHeadingRegex);
|
|
11255
|
+
cachedHeadingRe = await cachedHeadingPromise;
|
|
11256
|
+
};
|
|
11257
|
+
const isDocumentStructureHeading = (text) => {
|
|
11258
|
+
const re = cachedHeadingRe;
|
|
11259
|
+
if (!re) return false;
|
|
11260
|
+
return re.test(text);
|
|
11261
|
+
};
|
|
10374
11262
|
const EMPTY_GENERIC_ROLES$1 = /* @__PURE__ */ new Set();
|
|
10375
11263
|
/**
|
|
10376
11264
|
* Load generic-roles.json and cache the result on the
|
|
@@ -10402,7 +11290,7 @@ let _streetTypesPromise = null;
|
|
|
10402
11290
|
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
10403
11291
|
const loadStreetTypeRegex = async () => {
|
|
10404
11292
|
try {
|
|
10405
|
-
const data = (await
|
|
11293
|
+
const data = (await Promise.resolve().then(() => address_street_types_exports)).default ?? {};
|
|
10406
11294
|
const words = /* @__PURE__ */ new Set();
|
|
10407
11295
|
for (const [key, val] of Object.entries(data)) {
|
|
10408
11296
|
if (key.startsWith("_")) continue;
|
|
@@ -10465,6 +11353,7 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
10465
11353
|
if (STANDALONE_YEAR_RE.test(trimmed) && normalized.source !== "trigger") continue;
|
|
10466
11354
|
if (fullText && normalized.source !== "trigger" && /^\d/.test(trimmed) && NUMBER_ABBREV_RE.test(fullText.slice(Math.max(0, normalized.start - 10), normalized.start))) continue;
|
|
10467
11355
|
if (normalized.label === "registration number" && /^[\p{L}]{1,2}$/u.test(trimmed)) continue;
|
|
11356
|
+
if (normalized.label === "organization" && isDocumentStructureHeading(trimmed)) continue;
|
|
10468
11357
|
if (normalized.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
|
|
10469
11358
|
if (normalized.label === "person") {
|
|
10470
11359
|
const trimmedToken = trimmed.replace(/[.,;:!?]+$/u, "").trim();
|
|
@@ -10679,7 +11568,7 @@ let streetTypeReLoaded = false;
|
|
|
10679
11568
|
const loadStreetTypeRe = async () => {
|
|
10680
11569
|
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
10681
11570
|
try {
|
|
10682
|
-
const config = (await
|
|
11571
|
+
const config = (await Promise.resolve().then(() => address_street_types_exports)).default ?? {};
|
|
10683
11572
|
const words = [];
|
|
10684
11573
|
for (const value of Object.values(config)) {
|
|
10685
11574
|
if (!Array.isArray(value)) continue;
|
|
@@ -10756,7 +11645,55 @@ const SENTENCE_STARTER_WORDS = new Set([
|
|
|
10756
11645
|
]);
|
|
10757
11646
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
10758
11647
|
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
11648
|
+
const CURATED_PATTERN_SYNTAX_RE = /[|\\]/g;
|
|
11649
|
+
const stripCuratedPatternSyntax = (value) => value.includes("|") || value.includes("\\") ? value.replace(CURATED_PATTERN_SYNTAX_RE, "") : value;
|
|
10759
11650
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
11651
|
+
const EMPTY_PATTERN_LABELS = [];
|
|
11652
|
+
const EMPTY_PATTERN_SOURCES = [];
|
|
11653
|
+
const patternLabels = (labels) => {
|
|
11654
|
+
if (labels === void 0) return EMPTY_PATTERN_LABELS;
|
|
11655
|
+
return Array.isArray(labels) ? labels : [labels];
|
|
11656
|
+
};
|
|
11657
|
+
const patternSources = (sources) => {
|
|
11658
|
+
if (sources === void 0) return EMPTY_PATTERN_SOURCES;
|
|
11659
|
+
return Array.isArray(sources) ? sources : [sources];
|
|
11660
|
+
};
|
|
11661
|
+
const addPatternLabel = (list, index, label) => {
|
|
11662
|
+
const existing = list[index];
|
|
11663
|
+
if (existing === void 0) {
|
|
11664
|
+
list[index] = label;
|
|
11665
|
+
return;
|
|
11666
|
+
}
|
|
11667
|
+
if (Array.isArray(existing)) {
|
|
11668
|
+
if (!existing.includes(label)) existing.push(label);
|
|
11669
|
+
return;
|
|
11670
|
+
}
|
|
11671
|
+
if (existing !== label) list[index] = [existing, label];
|
|
11672
|
+
};
|
|
11673
|
+
const addPatternSource = (list, index, source) => {
|
|
11674
|
+
const existing = list[index];
|
|
11675
|
+
if (existing === void 0) {
|
|
11676
|
+
list[index] = source;
|
|
11677
|
+
return;
|
|
11678
|
+
}
|
|
11679
|
+
if (Array.isArray(existing)) {
|
|
11680
|
+
if (!existing.includes(source)) existing.push(source);
|
|
11681
|
+
return;
|
|
11682
|
+
}
|
|
11683
|
+
if (existing !== source) list[index] = [existing, source];
|
|
11684
|
+
};
|
|
11685
|
+
const addOptionalPatternLabel = (list, index, label) => {
|
|
11686
|
+
const existing = list[index];
|
|
11687
|
+
if (existing === void 0) {
|
|
11688
|
+
list[index] = label;
|
|
11689
|
+
return;
|
|
11690
|
+
}
|
|
11691
|
+
if (Array.isArray(existing)) {
|
|
11692
|
+
if (!existing.includes(label)) existing.push(label);
|
|
11693
|
+
return;
|
|
11694
|
+
}
|
|
11695
|
+
if (existing !== label) list[index] = [existing, label];
|
|
11696
|
+
};
|
|
10760
11697
|
const getCityEntries = (dictionaries, allowedCountries) => {
|
|
10761
11698
|
const byCountry = dictionaries?.citiesByCountry;
|
|
10762
11699
|
if (!byCountry) return dictionaries?.cities ?? [];
|
|
@@ -10809,7 +11746,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
10809
11746
|
const sourceList = [];
|
|
10810
11747
|
const patternIndex = /* @__PURE__ */ new Map();
|
|
10811
11748
|
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
10812
|
-
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry)
|
|
11749
|
+
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : stripCuratedPatternSyntax(normalizeForSearch(entry));
|
|
10813
11750
|
if (normalized.length === 0) return;
|
|
10814
11751
|
const lower = normalized.toLowerCase();
|
|
10815
11752
|
if (source !== "custom-deny-list" && label !== "address") {
|
|
@@ -10818,15 +11755,15 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
10818
11755
|
}
|
|
10819
11756
|
const existing = patternIndex.get(lower);
|
|
10820
11757
|
if (existing !== void 0) {
|
|
10821
|
-
|
|
10822
|
-
|
|
10823
|
-
if (source === "custom-deny-list" && !customLabelList[existing].includes(label)) customLabelList
|
|
11758
|
+
addPatternLabel(labelList, existing, label);
|
|
11759
|
+
addPatternSource(sourceList, existing, source);
|
|
11760
|
+
if (source === "custom-deny-list" && !patternLabels(customLabelList[existing]).includes(label)) addOptionalPatternLabel(customLabelList, existing, label);
|
|
10824
11761
|
} else {
|
|
10825
11762
|
patternIndex.set(lower, patternList.length);
|
|
10826
11763
|
patternList.push(normalized);
|
|
10827
|
-
labelList.push(
|
|
10828
|
-
|
|
10829
|
-
sourceList.push(
|
|
11764
|
+
labelList.push(label);
|
|
11765
|
+
if (source === "custom-deny-list") customLabelList[patternList.length - 1] = label;
|
|
11766
|
+
sourceList.push(source);
|
|
10830
11767
|
}
|
|
10831
11768
|
};
|
|
10832
11769
|
if (hasDenyList) {
|
|
@@ -10893,35 +11830,34 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
10893
11830
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
10894
11831
|
if (!config.enableNameCorpus || excludeCategories.has("Names")) return;
|
|
10895
11832
|
const addNameEntry = (name, source) => {
|
|
10896
|
-
const normalized = normalizeForSearch(name)
|
|
11833
|
+
const normalized = stripCuratedPatternSyntax(normalizeForSearch(name));
|
|
10897
11834
|
if (normalized.length === 0) return;
|
|
10898
11835
|
if (isCuratedNoiseAcronym(normalized)) return;
|
|
10899
11836
|
const lower = normalized.toLowerCase();
|
|
10900
11837
|
const existing = patternIndex.get(lower);
|
|
10901
11838
|
if (existing !== void 0) {
|
|
10902
|
-
|
|
10903
|
-
|
|
11839
|
+
addPatternLabel(labelList, existing, "person");
|
|
11840
|
+
addPatternSource(sourceList, existing, source);
|
|
10904
11841
|
} else {
|
|
10905
11842
|
patternIndex.set(lower, patternList.length);
|
|
10906
11843
|
patternList.push(normalized);
|
|
10907
|
-
labelList.push(
|
|
10908
|
-
sourceList.push(
|
|
11844
|
+
labelList.push("person");
|
|
11845
|
+
sourceList.push(source);
|
|
10909
11846
|
}
|
|
10910
11847
|
};
|
|
10911
11848
|
for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
|
|
10912
11849
|
for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
|
|
10913
11850
|
for (const title of getNameCorpusTitles(ctx)) {
|
|
10914
|
-
const norm = normalizeForSearch(title)
|
|
11851
|
+
const norm = stripCuratedPatternSyntax(normalizeForSearch(title));
|
|
10915
11852
|
if (norm.length === 0) continue;
|
|
10916
11853
|
const lower = norm.toLowerCase();
|
|
10917
11854
|
const existing = patternIndex.get(lower);
|
|
10918
|
-
if (existing !== void 0)
|
|
10919
|
-
|
|
10920
|
-
} else {
|
|
11855
|
+
if (existing !== void 0) addPatternSource(sourceList, existing, "title");
|
|
11856
|
+
else {
|
|
10921
11857
|
patternIndex.set(lower, patternList.length);
|
|
10922
11858
|
patternList.push(norm);
|
|
10923
|
-
labelList.push(
|
|
10924
|
-
sourceList.push(
|
|
11859
|
+
labelList.push("person");
|
|
11860
|
+
sourceList.push("title");
|
|
10925
11861
|
}
|
|
10926
11862
|
}
|
|
10927
11863
|
};
|
|
@@ -10972,18 +11908,18 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
10972
11908
|
const idx = match.pattern;
|
|
10973
11909
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
10974
11910
|
const localIdx = idx - sliceStart;
|
|
10975
|
-
const sources = data.sources[localIdx]
|
|
11911
|
+
const sources = patternSources(data.sources[localIdx]);
|
|
10976
11912
|
const matchText = fullText.slice(match.start, match.end);
|
|
10977
11913
|
const sourceChar = fullText[match.start] ?? "";
|
|
10978
11914
|
const keyword = matchText.toLowerCase();
|
|
10979
|
-
const labels = data.labels[localIdx];
|
|
11915
|
+
const labels = patternLabels(data.labels[localIdx]);
|
|
10980
11916
|
const pattern = data.originals[localIdx] ?? "";
|
|
10981
|
-
const customPatternLabels = data.customLabels[localIdx]
|
|
11917
|
+
const customPatternLabels = patternLabels(data.customLabels[localIdx]);
|
|
10982
11918
|
const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
|
|
10983
11919
|
const customLabels = customEdgesAreValid ? customPatternLabels : [];
|
|
10984
|
-
if (
|
|
11920
|
+
if (labels.length === 0 && customLabels.length === 0) continue;
|
|
10985
11921
|
const acronymMatchesAcronym = !(pattern.length > 0 && pattern.length <= 5 && ALL_UPPER_RE.test(pattern)) || ALL_UPPER_RE.test(matchText);
|
|
10986
|
-
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && acronymMatchesAcronym && !ALL_UPPER_RE.test(matchText) ?
|
|
11922
|
+
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && acronymMatchesAcronym && !ALL_UPPER_RE.test(matchText) ? labels.filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
|
|
10987
11923
|
const filteredCuratedLabels = isDottedAcronymSuffixCollision(fullText, match.start, matchText) ? [] : curatedLabels;
|
|
10988
11924
|
if (filteredCuratedLabels.length === 0 && customLabels.length === 0) continue;
|
|
10989
11925
|
const entry = {
|
|
@@ -11296,7 +12232,7 @@ let cachedBrCepContextPromise = null;
|
|
|
11296
12232
|
const loadBrCueWords = async () => {
|
|
11297
12233
|
const sources = await Promise.all([(async () => {
|
|
11298
12234
|
try {
|
|
11299
|
-
return (await
|
|
12235
|
+
return (await Promise.resolve().then(() => address_street_types_exports)).default["pt-br"];
|
|
11300
12236
|
} catch {
|
|
11301
12237
|
return;
|
|
11302
12238
|
}
|
|
@@ -11404,10 +12340,11 @@ const getBoundaryRe = async () => {
|
|
|
11404
12340
|
* Returns string[] for the unified TextSearch
|
|
11405
12341
|
* builder. Empty if data package is not installed.
|
|
11406
12342
|
*/
|
|
11407
|
-
|
|
11408
|
-
|
|
12343
|
+
let streetTypePatternsPromise = null;
|
|
12344
|
+
const loadStreetTypePatterns = async () => {
|
|
12345
|
+
let config;
|
|
11409
12346
|
try {
|
|
11410
|
-
config = (await
|
|
12347
|
+
config = (await Promise.resolve().then(() => address_street_types_exports)).default;
|
|
11411
12348
|
} catch {
|
|
11412
12349
|
return [];
|
|
11413
12350
|
}
|
|
@@ -11418,6 +12355,10 @@ const buildStreetTypePatterns = async () => {
|
|
|
11418
12355
|
}
|
|
11419
12356
|
return words;
|
|
11420
12357
|
};
|
|
12358
|
+
const buildStreetTypePatterns = async () => {
|
|
12359
|
+
streetTypePatternsPromise ??= loadStreetTypePatterns();
|
|
12360
|
+
return streetTypePatternsPromise;
|
|
12361
|
+
};
|
|
11421
12362
|
const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntities, brCepContextRe) => {
|
|
11422
12363
|
const seeds = [];
|
|
11423
12364
|
for (const match of allMatches) {
|
|
@@ -11607,6 +12548,35 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
11607
12548
|
end: Math.max(rightPos, end)
|
|
11608
12549
|
};
|
|
11609
12550
|
};
|
|
12551
|
+
const STREET_SEED_TYPES = new Set(["street-word", "house-number"]);
|
|
12552
|
+
const DESTINATION_SEED_TYPES = new Set(["postal-code", "city"]);
|
|
12553
|
+
const resolveNewlineBoundary = (spanStart, text, cluster) => {
|
|
12554
|
+
const newlines = (text.match(/\n/gu) ?? []).length;
|
|
12555
|
+
if (newlines === 0) return { kind: "keep" };
|
|
12556
|
+
if (newlines > 1) return { kind: "drop" };
|
|
12557
|
+
const relativeNewline = text.indexOf("\n");
|
|
12558
|
+
const newlineAbs = spanStart + relativeNewline;
|
|
12559
|
+
let streetAbove = false;
|
|
12560
|
+
let streetBelow = false;
|
|
12561
|
+
let destAbove = false;
|
|
12562
|
+
let destBelow = false;
|
|
12563
|
+
for (const seed of cluster.seeds) {
|
|
12564
|
+
const isAbove = seed.end <= newlineAbs;
|
|
12565
|
+
const isStreet = STREET_SEED_TYPES.has(seed.type);
|
|
12566
|
+
const isDest = DESTINATION_SEED_TYPES.has(seed.type);
|
|
12567
|
+
if (isStreet && isAbove) streetAbove = true;
|
|
12568
|
+
if (isStreet && !isAbove) streetBelow = true;
|
|
12569
|
+
if (isDest && isAbove) destAbove = true;
|
|
12570
|
+
if (isDest && !isAbove) destBelow = true;
|
|
12571
|
+
}
|
|
12572
|
+
if (streetAbove && destBelow || streetBelow && destAbove) return { kind: "keep" };
|
|
12573
|
+
if (streetAbove && destAbove) return {
|
|
12574
|
+
kind: "trim",
|
|
12575
|
+
relativeEnd: relativeNewline
|
|
12576
|
+
};
|
|
12577
|
+
return { kind: "drop" };
|
|
12578
|
+
};
|
|
12579
|
+
const normaliseLineBreaks = (text) => text.replace(/\r\n?/gu, "\n");
|
|
11610
12580
|
/**
|
|
11611
12581
|
* Process address seeds from the unified search.
|
|
11612
12582
|
* Receives all matches; filters to the street types
|
|
@@ -11624,14 +12594,16 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
|
|
|
11624
12594
|
const score = scoreCluster(cluster);
|
|
11625
12595
|
if (score < .6) continue;
|
|
11626
12596
|
const { start, end } = await expandCluster(fullText, cluster, existingEntities);
|
|
11627
|
-
const
|
|
11628
|
-
|
|
11629
|
-
if (
|
|
12597
|
+
const rawText = fullText.slice(start, end);
|
|
12598
|
+
const resolution = resolveNewlineBoundary(start, normaliseLineBreaks(rawText), cluster);
|
|
12599
|
+
if (resolution.kind === "drop") continue;
|
|
12600
|
+
const effectiveText = resolution.kind === "trim" ? rawText.slice(0, resolution.relativeEnd).trim() : rawText.trim();
|
|
12601
|
+
if (effectiveText.length < 5 || effectiveText.length > 300) continue;
|
|
11630
12602
|
results.push({
|
|
11631
12603
|
start,
|
|
11632
|
-
end: start +
|
|
12604
|
+
end: start + effectiveText.length,
|
|
11633
12605
|
label: "address",
|
|
11634
|
-
text,
|
|
12606
|
+
text: effectiveText,
|
|
11635
12607
|
score,
|
|
11636
12608
|
source: DETECTION_SOURCES.REGEX
|
|
11637
12609
|
});
|
|
@@ -11713,6 +12685,246 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
11713
12685
|
return results;
|
|
11714
12686
|
};
|
|
11715
12687
|
//#endregion
|
|
12688
|
+
//#region src/data/address-street-types.json
|
|
12689
|
+
var address_street_types_exports = /* @__PURE__ */ __exportAll({
|
|
12690
|
+
_comment: () => _comment,
|
|
12691
|
+
cs: () => cs,
|
|
12692
|
+
de: () => de$1,
|
|
12693
|
+
default: () => address_street_types_default,
|
|
12694
|
+
en: () => en,
|
|
12695
|
+
es: () => es$1,
|
|
12696
|
+
fr: () => fr$1,
|
|
12697
|
+
hu: () => hu$1,
|
|
12698
|
+
it: () => it$1,
|
|
12699
|
+
nl: () => nl$1,
|
|
12700
|
+
pl: () => pl$1,
|
|
12701
|
+
pt: () => pt$1,
|
|
12702
|
+
"pt-br": () => pt_br,
|
|
12703
|
+
ro: () => ro$1,
|
|
12704
|
+
sk: () => sk$1,
|
|
12705
|
+
sv: () => sv
|
|
12706
|
+
});
|
|
12707
|
+
var _comment = "Street type words by language. Used by address seed detector to identify street names via Aho-Corasick.";
|
|
12708
|
+
var cs = [
|
|
12709
|
+
"alej",
|
|
12710
|
+
"bulvár",
|
|
12711
|
+
"nábř.",
|
|
12712
|
+
"nábřeží",
|
|
12713
|
+
"nám.",
|
|
12714
|
+
"náměstí",
|
|
12715
|
+
"sady",
|
|
12716
|
+
"tř.",
|
|
12717
|
+
"třída",
|
|
12718
|
+
"ul.",
|
|
12719
|
+
"ulice"
|
|
12720
|
+
];
|
|
12721
|
+
var de$1 = [
|
|
12722
|
+
"Allee",
|
|
12723
|
+
"Gasse",
|
|
12724
|
+
"Platz",
|
|
12725
|
+
"Ring",
|
|
12726
|
+
"Straße",
|
|
12727
|
+
"Strasse",
|
|
12728
|
+
"Weg"
|
|
12729
|
+
];
|
|
12730
|
+
var en = [
|
|
12731
|
+
"Ave.",
|
|
12732
|
+
"Avenue",
|
|
12733
|
+
"Blvd.",
|
|
12734
|
+
"Boulevard",
|
|
12735
|
+
"Circle",
|
|
12736
|
+
"Court",
|
|
12737
|
+
"Crescent",
|
|
12738
|
+
"Drive",
|
|
12739
|
+
"Highway",
|
|
12740
|
+
"Lane",
|
|
12741
|
+
"Parkway",
|
|
12742
|
+
"Place",
|
|
12743
|
+
"Rd.",
|
|
12744
|
+
"Road",
|
|
12745
|
+
"Square",
|
|
12746
|
+
"St.",
|
|
12747
|
+
"Street",
|
|
12748
|
+
"Terrace",
|
|
12749
|
+
"Way"
|
|
12750
|
+
];
|
|
12751
|
+
var es$1 = [
|
|
12752
|
+
"Av.",
|
|
12753
|
+
"Avda.",
|
|
12754
|
+
"avenida",
|
|
12755
|
+
"bulevar",
|
|
12756
|
+
"C/",
|
|
12757
|
+
"calle",
|
|
12758
|
+
"camino",
|
|
12759
|
+
"carrera",
|
|
12760
|
+
"carretera",
|
|
12761
|
+
"Cra.",
|
|
12762
|
+
"Ctra.",
|
|
12763
|
+
"glorieta",
|
|
12764
|
+
"paseo",
|
|
12765
|
+
"plaza",
|
|
12766
|
+
"Pza.",
|
|
12767
|
+
"ronda",
|
|
12768
|
+
"travesía"
|
|
12769
|
+
];
|
|
12770
|
+
var fr$1 = [
|
|
12771
|
+
"allée",
|
|
12772
|
+
"av",
|
|
12773
|
+
"av.",
|
|
12774
|
+
"avenue",
|
|
12775
|
+
"bd",
|
|
12776
|
+
"bd.",
|
|
12777
|
+
"boulevard",
|
|
12778
|
+
"chemin",
|
|
12779
|
+
"cours",
|
|
12780
|
+
"impasse",
|
|
12781
|
+
"passage",
|
|
12782
|
+
"pl",
|
|
12783
|
+
"pl.",
|
|
12784
|
+
"place",
|
|
12785
|
+
"quai",
|
|
12786
|
+
"résidence",
|
|
12787
|
+
"route",
|
|
12788
|
+
"rte",
|
|
12789
|
+
"rte.",
|
|
12790
|
+
"rue",
|
|
12791
|
+
"ruelle",
|
|
12792
|
+
"square",
|
|
12793
|
+
"voie"
|
|
12794
|
+
];
|
|
12795
|
+
var hu$1 = [
|
|
12796
|
+
"fasor",
|
|
12797
|
+
"körút",
|
|
12798
|
+
"köz",
|
|
12799
|
+
"krt.",
|
|
12800
|
+
"rakpart",
|
|
12801
|
+
"sétány",
|
|
12802
|
+
"sor",
|
|
12803
|
+
"tér",
|
|
12804
|
+
"u.",
|
|
12805
|
+
"út",
|
|
12806
|
+
"utca"
|
|
12807
|
+
];
|
|
12808
|
+
var it$1 = [
|
|
12809
|
+
"borgo",
|
|
12810
|
+
"calle",
|
|
12811
|
+
"contrada",
|
|
12812
|
+
"corso",
|
|
12813
|
+
"largo",
|
|
12814
|
+
"lungarno",
|
|
12815
|
+
"lungomare",
|
|
12816
|
+
"lungotevere",
|
|
12817
|
+
"passeggiata",
|
|
12818
|
+
"piazza",
|
|
12819
|
+
"piazzale",
|
|
12820
|
+
"salita",
|
|
12821
|
+
"strada",
|
|
12822
|
+
"stradone",
|
|
12823
|
+
"traversa",
|
|
12824
|
+
"via",
|
|
12825
|
+
"viale",
|
|
12826
|
+
"vico",
|
|
12827
|
+
"vicolo",
|
|
12828
|
+
"v.le",
|
|
12829
|
+
"v.lo"
|
|
12830
|
+
];
|
|
12831
|
+
var nl$1 = [
|
|
12832
|
+
"gracht",
|
|
12833
|
+
"kade",
|
|
12834
|
+
"laan",
|
|
12835
|
+
"plein",
|
|
12836
|
+
"straat",
|
|
12837
|
+
"weg"
|
|
12838
|
+
];
|
|
12839
|
+
var pl$1 = [
|
|
12840
|
+
"al.",
|
|
12841
|
+
"aleja",
|
|
12842
|
+
"aleje",
|
|
12843
|
+
"aleją",
|
|
12844
|
+
"alei",
|
|
12845
|
+
"bulwar",
|
|
12846
|
+
"os.",
|
|
12847
|
+
"osiedle",
|
|
12848
|
+
"pl.",
|
|
12849
|
+
"plac",
|
|
12850
|
+
"placu",
|
|
12851
|
+
"rondo",
|
|
12852
|
+
"rynek",
|
|
12853
|
+
"skwer",
|
|
12854
|
+
"ul.",
|
|
12855
|
+
"ulica",
|
|
12856
|
+
"ulicy"
|
|
12857
|
+
];
|
|
12858
|
+
var pt$1 = [
|
|
12859
|
+
"avenida",
|
|
12860
|
+
"largo",
|
|
12861
|
+
"praça",
|
|
12862
|
+
"rua",
|
|
12863
|
+
"travessa"
|
|
12864
|
+
];
|
|
12865
|
+
var pt_br = [
|
|
12866
|
+
"alameda",
|
|
12867
|
+
"av.",
|
|
12868
|
+
"avenida",
|
|
12869
|
+
"estrada",
|
|
12870
|
+
"largo",
|
|
12871
|
+
"praça",
|
|
12872
|
+
"praca",
|
|
12873
|
+
"rodovia",
|
|
12874
|
+
"rua",
|
|
12875
|
+
"travessa"
|
|
12876
|
+
];
|
|
12877
|
+
var ro$1 = [
|
|
12878
|
+
"aleea",
|
|
12879
|
+
"bd.",
|
|
12880
|
+
"bulevardul",
|
|
12881
|
+
"calea",
|
|
12882
|
+
"fundătura",
|
|
12883
|
+
"intrare",
|
|
12884
|
+
"piața",
|
|
12885
|
+
"șoseaua",
|
|
12886
|
+
"splaiul",
|
|
12887
|
+
"str.",
|
|
12888
|
+
"strada"
|
|
12889
|
+
];
|
|
12890
|
+
var sk$1 = [
|
|
12891
|
+
"alej",
|
|
12892
|
+
"nábr.",
|
|
12893
|
+
"nábrežie",
|
|
12894
|
+
"nám.",
|
|
12895
|
+
"námest.",
|
|
12896
|
+
"námestie",
|
|
12897
|
+
"sady",
|
|
12898
|
+
"tr.",
|
|
12899
|
+
"trieda",
|
|
12900
|
+
"ul.",
|
|
12901
|
+
"ulica"
|
|
12902
|
+
];
|
|
12903
|
+
var sv = [
|
|
12904
|
+
"allé",
|
|
12905
|
+
"backe",
|
|
12906
|
+
"gata",
|
|
12907
|
+
"gränd",
|
|
12908
|
+
"torg"
|
|
12909
|
+
];
|
|
12910
|
+
var address_street_types_default = {
|
|
12911
|
+
_comment,
|
|
12912
|
+
cs,
|
|
12913
|
+
de: de$1,
|
|
12914
|
+
en,
|
|
12915
|
+
es: es$1,
|
|
12916
|
+
fr: fr$1,
|
|
12917
|
+
hu: hu$1,
|
|
12918
|
+
it: it$1,
|
|
12919
|
+
nl: nl$1,
|
|
12920
|
+
pl: pl$1,
|
|
12921
|
+
pt: pt$1,
|
|
12922
|
+
"pt-br": pt_br,
|
|
12923
|
+
ro: ro$1,
|
|
12924
|
+
sk: sk$1,
|
|
12925
|
+
sv
|
|
12926
|
+
};
|
|
12927
|
+
//#endregion
|
|
11716
12928
|
//#region src/filters/confidence-boost.ts
|
|
11717
12929
|
const BARE_STOPWORDS = new Set([
|
|
11718
12930
|
"Příloha",
|
|
@@ -11863,7 +13075,7 @@ let _streetAbbrevs = buildStreetAbbrevs(address_street_types_default);
|
|
|
11863
13075
|
let _streetAbbrevsPromise = null;
|
|
11864
13076
|
const loadStreetAbbrevs = async () => {
|
|
11865
13077
|
try {
|
|
11866
|
-
const mod = await
|
|
13078
|
+
const mod = await Promise.resolve().then(() => address_street_types_exports);
|
|
11867
13079
|
_streetAbbrevs = buildStreetAbbrevs(mod.default ?? mod);
|
|
11868
13080
|
} catch {
|
|
11869
13081
|
_streetAbbrevs ??= /* @__PURE__ */ new Set();
|
|
@@ -11939,7 +13151,10 @@ const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
|
11939
13151
|
if (streetText.length < 4) continue;
|
|
11940
13152
|
if (hasTemporalPrep) continue;
|
|
11941
13153
|
if (existingEntities.some((e) => e.start <= streetStart && e.end >= numEnd)) continue;
|
|
11942
|
-
const
|
|
13154
|
+
const hasColon = fullText.slice(Math.max(0, streetStart - 5), streetStart).includes(":");
|
|
13155
|
+
let score = .8;
|
|
13156
|
+
if (hasColon) score = .95;
|
|
13157
|
+
else if (inHeader) score = .85;
|
|
11943
13158
|
results.push({
|
|
11944
13159
|
start: streetStart,
|
|
11945
13160
|
end: numEnd,
|
|
@@ -12352,10 +13567,10 @@ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-lis
|
|
|
12352
13567
|
/**
|
|
12353
13568
|
* Characters allowed in the gap between two adjacent
|
|
12354
13569
|
* same-label entities that should be merged: spaces,
|
|
12355
|
-
* tabs, commas, and hyphens. Uses `[ \t
|
|
13570
|
+
* tabs, commas, and hyphens. Uses `[ \t,-]` instead
|
|
12356
13571
|
* of `\s` to avoid merging entities across newlines.
|
|
12357
13572
|
*/
|
|
12358
|
-
const GAP_PATTERN = /^[ \t
|
|
13573
|
+
const GAP_PATTERN = /^[ \t,-]+$/;
|
|
12359
13574
|
const isLegalFormOrganization = (entity) => entity.label === "organization" && entity.source === DETECTION_SOURCES.LEGAL_FORM;
|
|
12360
13575
|
/**
|
|
12361
13576
|
* Build a set of word boundary offsets for the full
|
|
@@ -12496,7 +13711,8 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
12496
13711
|
break;
|
|
12497
13712
|
}
|
|
12498
13713
|
}
|
|
12499
|
-
|
|
13714
|
+
const isMergeableGap = gap.length === 0 || gap.length <= MAX_GAP && GAP_PATTERN.test(gap);
|
|
13715
|
+
if (!hasLockedBoundary$1(prev) && !((isLegalFormOrganization(prev) || isLegalFormOrganization(entity)) && gap.includes(",")) && entity.label !== "country" && !gapOccupied && isMergeableGap) {
|
|
12500
13716
|
prev.end = entity.end;
|
|
12501
13717
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
12502
13718
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -12529,6 +13745,7 @@ const fixPartialWords = (entities, fullText) => {
|
|
|
12529
13745
|
const endPositions = byEnd.map((x) => x.entity.end);
|
|
12530
13746
|
return sorted.map((e, eIdx) => {
|
|
12531
13747
|
if (hasLockedBoundary$1(e)) return e;
|
|
13748
|
+
if (e.text !== fullText.slice(e.start, e.end)) return e;
|
|
12532
13749
|
let newStart = wordStartAt(e.start, boundaries, fullText);
|
|
12533
13750
|
let newEnd = wordEndAt(e.end, boundaries, fullText);
|
|
12534
13751
|
let lo = 0;
|
|
@@ -12657,21 +13874,21 @@ const ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
|
12657
13874
|
const createAllowedLabelSet$1 = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
12658
13875
|
const labelIsAllowed$1 = (label, allowedLabels) => allowedLabels === null || allowedLabels.has(label);
|
|
12659
13876
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
12660
|
-
|
|
13877
|
+
isLegalFormsEnabled(config);
|
|
12661
13878
|
const allowedLabels = createAllowedLabelSet$1(config.enableHotwordRules === true ? expandLabelsForHotwordRules(config.labels) : config.labels);
|
|
12662
13879
|
const customRegexes = config.enableRegex ? (config.customRegexes ?? []).filter((entry) => labelIsAllowed$1(entry.label, allowedLabels)) : [];
|
|
12663
|
-
const [
|
|
12664
|
-
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
13880
|
+
const [triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
12665
13881
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
12666
13882
|
patterns: [],
|
|
12667
13883
|
rules: []
|
|
12668
13884
|
}),
|
|
12669
13885
|
config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
|
|
12670
13886
|
buildStreetTypePatterns(),
|
|
12671
|
-
config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ?
|
|
13887
|
+
config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ? getCurrencyPatternEntries() : Promise.resolve([]),
|
|
12672
13888
|
config.enableRegex && labelIsAllowed$1("date", allowedLabels) ? getDatePatterns() : Promise.resolve([]),
|
|
12673
13889
|
config.enableRegex && labelIsAllowed$1("address", allowedLabels) ? getSigningClausePatterns() : Promise.resolve([])
|
|
12674
13890
|
]);
|
|
13891
|
+
const legalForms = [];
|
|
12675
13892
|
const allRegex = [];
|
|
12676
13893
|
const regexMeta = [];
|
|
12677
13894
|
if (config.enableRegex) for (const [index, pattern] of REGEX_PATTERNS.entries()) {
|
|
@@ -12751,7 +13968,6 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
12751
13968
|
start: offset,
|
|
12752
13969
|
end: offset + (countryResult?.patterns.length ?? 0)
|
|
12753
13970
|
};
|
|
12754
|
-
offset = countriesSlice.end;
|
|
12755
13971
|
const wrapWholeWord = (s, wholeWords) => ({
|
|
12756
13972
|
pattern: s,
|
|
12757
13973
|
literal: true,
|
|
@@ -12762,7 +13978,18 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
12762
13978
|
const last = pattern.at(-1) ?? "";
|
|
12763
13979
|
return ALNUM_RE.test(first) && ALNUM_RE.test(last);
|
|
12764
13980
|
};
|
|
12765
|
-
const
|
|
13981
|
+
const literalPatternText = (entry) => {
|
|
13982
|
+
if (typeof entry === "string") return entry;
|
|
13983
|
+
if (entry instanceof RegExp) throw new Error("Expected literal country pattern, got RegExp");
|
|
13984
|
+
if (entry.pattern instanceof RegExp) throw new Error("Expected literal country pattern, got RegExp entry");
|
|
13985
|
+
return entry.pattern;
|
|
13986
|
+
};
|
|
13987
|
+
const canUseGlobalWholeWordLiterals = !(denyListData?.sources.some((sources) => sources.includes("custom-deny-list")) ?? false) && gazResult === null;
|
|
13988
|
+
const literalAllPatterns = canUseGlobalWholeWordLiterals ? [
|
|
13989
|
+
...denyListOriginals,
|
|
13990
|
+
...streetTypes,
|
|
13991
|
+
...countryResult?.patterns.map(literalPatternText) ?? []
|
|
13992
|
+
] : [
|
|
12766
13993
|
...denyListOriginals.map((pattern, index) => wrapWholeWord(pattern, (denyListData?.sources[index] ?? []).includes("custom-deny-list") ? customDenyListNeedsWholeWords(pattern) : true)),
|
|
12767
13994
|
...streetTypes.map((pattern) => wrapWholeWord(pattern, true)),
|
|
12768
13995
|
...gazResult?.patterns ?? [],
|
|
@@ -12772,6 +13999,10 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
12772
13999
|
tsRegex,
|
|
12773
14000
|
tsCustomRegex,
|
|
12774
14001
|
tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
|
|
14002
|
+
...canUseGlobalWholeWordLiterals ? {
|
|
14003
|
+
allLiteral: true,
|
|
14004
|
+
wholeWords: true
|
|
14005
|
+
} : {},
|
|
12775
14006
|
caseInsensitive: true,
|
|
12776
14007
|
overlapStrategy: "all"
|
|
12777
14008
|
}) : new (getTextSearch())([]),
|
|
@@ -12917,6 +14148,7 @@ const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
|
12917
14148
|
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
12918
14149
|
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
12919
14150
|
const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
|
|
14151
|
+
const BARE_POSTAL_CODE_RE = /^\s*(?:\d{3}\s?\d{2}|\d{2}[-–]\d{3}|\d{5}(?:[-–]\d{3,4})?)\s*$/u;
|
|
12920
14152
|
const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
|
|
12921
14153
|
const shouldReplace = (a, b) => {
|
|
12922
14154
|
const aLen = a.end - a.start;
|
|
@@ -12925,6 +14157,19 @@ const shouldReplace = (a, b) => {
|
|
|
12925
14157
|
if (aCallerOwned !== isCallerOwnedEntity(b)) return aCallerOwned;
|
|
12926
14158
|
if (a.label === b.label && LITERAL_SOURCES.has(a.source) && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
|
|
12927
14159
|
if (a.label === b.label && LITERAL_SOURCES.has(b.source) && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
|
|
14160
|
+
if (a.label === "address" && b.label === "address" && a.start <= b.start && a.end >= b.end && aLen > bLen && BARE_POSTAL_CODE_RE.test(b.text)) return true;
|
|
14161
|
+
if (a.label === "address" && b.label === "address" && b.start <= a.start && b.end >= a.end && bLen > aLen && BARE_POSTAL_CODE_RE.test(a.text)) return false;
|
|
14162
|
+
if (a.label === b.label && a.source === DETECTION_SOURCES.LEGAL_FORM && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
|
|
14163
|
+
if (a.label === b.label && b.source === DETECTION_SOURCES.LEGAL_FORM && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
|
|
14164
|
+
const LONGEST_WINS_LABELS = new Set([
|
|
14165
|
+
"date",
|
|
14166
|
+
"date of birth",
|
|
14167
|
+
"monetary amount",
|
|
14168
|
+
"phone number",
|
|
14169
|
+
"email address",
|
|
14170
|
+
"url"
|
|
14171
|
+
]);
|
|
14172
|
+
if (a.label === b.label && a.start === b.start && aLen !== bLen && LONGEST_WINS_LABELS.has(a.label)) return aLen > bLen;
|
|
12928
14173
|
if (a.label === "country" && (b.label === "person" || b.label === "organization") && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
|
|
12929
14174
|
if (b.label === "country" && (a.label === "person" || a.label === "organization") && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
|
|
12930
14175
|
const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
|
|
@@ -13080,6 +14325,16 @@ const TRAILING_PUNCT_CLASS = `["“”‘’'»!?]`;
|
|
|
13080
14325
|
* intentionally captured.
|
|
13081
14326
|
*/
|
|
13082
14327
|
const LEADING_PUNCT_CLASS = `["“”‘’'«¿¡]`;
|
|
14328
|
+
const LEADING_ELLIPSIS_RE = /^(?:\.{2,}|…+)/u;
|
|
14329
|
+
const ELLIPSIS_PREFIX_LABELS = new Set([
|
|
14330
|
+
"date",
|
|
14331
|
+
"date of birth",
|
|
14332
|
+
"monetary amount",
|
|
14333
|
+
"phone number",
|
|
14334
|
+
"email address",
|
|
14335
|
+
"url",
|
|
14336
|
+
"time"
|
|
14337
|
+
]);
|
|
13083
14338
|
const STRIP_BY_LABEL = {
|
|
13084
14339
|
colon: /[\s,;]+/,
|
|
13085
14340
|
default: /[\s:,;]+/
|
|
@@ -13098,7 +14353,7 @@ const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
|
13098
14353
|
const stripKind = COLON_LABELS.has(e.label) ? "colon" : "default";
|
|
13099
14354
|
const leadRe = LEADING_TRIM_BY_LABEL[stripKind];
|
|
13100
14355
|
const trailRe = TRAILING_TRIM_BY_LABEL[stripKind];
|
|
13101
|
-
const leadTrimmed = e.text.replace(leadRe, "");
|
|
14356
|
+
const leadTrimmed = (ELLIPSIS_PREFIX_LABELS.has(e.label) ? e.text.replace(LEADING_ELLIPSIS_RE, "") : e.text).replace(leadRe, "");
|
|
13102
14357
|
const lead = e.text.length - leadTrimmed.length;
|
|
13103
14358
|
let cleaned = leadTrimmed.replace(trailRe, "");
|
|
13104
14359
|
if (PERIOD_STRIPPED_LABELS.has(e.label) && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
@@ -13126,7 +14381,7 @@ const mergeAndDedup = (...layers) => {
|
|
|
13126
14381
|
const merged = [{ ...first }];
|
|
13127
14382
|
for (let i = 1; i < sorted.length; i++) {
|
|
13128
14383
|
const entity = sorted[i];
|
|
13129
|
-
const last = merged
|
|
14384
|
+
const last = merged.at(-1);
|
|
13130
14385
|
if (!entity || !last) continue;
|
|
13131
14386
|
if (last.end <= entity.start) {
|
|
13132
14387
|
merged.push({ ...entity });
|
|
@@ -13161,7 +14416,7 @@ let amountWordsLoaded = false;
|
|
|
13161
14416
|
const getAmountWordsRe = async () => {
|
|
13162
14417
|
if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
|
|
13163
14418
|
try {
|
|
13164
|
-
const alt = (await
|
|
14419
|
+
const alt = (await Promise.resolve().then(() => amount_words_exports)).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
13165
14420
|
amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
|
|
13166
14421
|
} catch {
|
|
13167
14422
|
amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
|
|
@@ -13181,6 +14436,45 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
|
|
|
13181
14436
|
text: fullText.slice(e.start, newEnd)
|
|
13182
14437
|
};
|
|
13183
14438
|
});
|
|
14439
|
+
let monetaryTrailingCurrencyRe = null;
|
|
14440
|
+
let monetaryTrailingCurrencyLoaded = false;
|
|
14441
|
+
const getMonetaryTrailingCurrencyRe = async () => {
|
|
14442
|
+
if (monetaryTrailingCurrencyLoaded) return monetaryTrailingCurrencyRe;
|
|
14443
|
+
try {
|
|
14444
|
+
const mod = await import("./currencies.mjs");
|
|
14445
|
+
const data = mod.default ?? mod;
|
|
14446
|
+
const codes = (data.codes ?? []).filter((c) => /^[A-Z]{2,4}$/.test(c));
|
|
14447
|
+
const names = (data.localNames ?? []).filter((n) => n.length > 0);
|
|
14448
|
+
const parts = [];
|
|
14449
|
+
if (names.length > 0) parts.push(names.map(escapeRegex).join("|"));
|
|
14450
|
+
if (codes.length > 0) parts.push(codes.map(escapeRegex).join("|"));
|
|
14451
|
+
if (parts.length === 0) monetaryTrailingCurrencyRe = null;
|
|
14452
|
+
else {
|
|
14453
|
+
const alt = parts.join("|");
|
|
14454
|
+
monetaryTrailingCurrencyRe = new RegExp(`^([^\\S\\n\\t]{0,4})(${alt})(?![\\p{L}\\p{N}])`, "u");
|
|
14455
|
+
}
|
|
14456
|
+
} catch {
|
|
14457
|
+
monetaryTrailingCurrencyRe = null;
|
|
14458
|
+
}
|
|
14459
|
+
monetaryTrailingCurrencyLoaded = true;
|
|
14460
|
+
return monetaryTrailingCurrencyRe;
|
|
14461
|
+
};
|
|
14462
|
+
const extendMonetaryTrailingCurrency = (entities, fullText, re) => {
|
|
14463
|
+
if (!re) return entities;
|
|
14464
|
+
return entities.map((e) => {
|
|
14465
|
+
if (e.label !== "monetary amount" || isCallerOwnedEntity(e)) return e;
|
|
14466
|
+
if (/\p{L}/u.test(e.text.slice(-1) ?? "")) return e;
|
|
14467
|
+
const after = fullText.slice(e.end);
|
|
14468
|
+
const m = re.exec(after);
|
|
14469
|
+
if (!m) return e;
|
|
14470
|
+
const newEnd = e.end + m[0].length;
|
|
14471
|
+
return {
|
|
14472
|
+
...e,
|
|
14473
|
+
end: newEnd,
|
|
14474
|
+
text: fullText.slice(e.start, newEnd)
|
|
14475
|
+
};
|
|
14476
|
+
});
|
|
14477
|
+
};
|
|
13184
14478
|
const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
13185
14479
|
const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
|
|
13186
14480
|
const DEFAULT_CUSTOM_REGEX_SCORE = .9;
|
|
@@ -13212,20 +14506,54 @@ const configKey = (config, gazetteerEntries) => {
|
|
|
13212
14506
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
13213
14507
|
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.nameCorpusLanguages?.toSorted().join(",") ?? ""}:${config.enableRegex}:${config.labels.toSorted().join(",")}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}:${config.enableCountries !== false}`;
|
|
13214
14508
|
};
|
|
14509
|
+
const sharedSearchByDictionaries = /* @__PURE__ */ new WeakMap();
|
|
14510
|
+
const sharedSearchWithoutDictionaries = /* @__PURE__ */ new Map();
|
|
14511
|
+
const sharedSearchCacheFor = (dictionaries) => {
|
|
14512
|
+
if (dictionaries === void 0) return sharedSearchWithoutDictionaries;
|
|
14513
|
+
const cached = sharedSearchByDictionaries.get(dictionaries);
|
|
14514
|
+
if (cached !== void 0) return cached;
|
|
14515
|
+
const created = /* @__PURE__ */ new Map();
|
|
14516
|
+
sharedSearchByDictionaries.set(dictionaries, created);
|
|
14517
|
+
return created;
|
|
14518
|
+
};
|
|
14519
|
+
const ensureSearchSupportData = async (config, ctx) => {
|
|
14520
|
+
if (!config.enableDenyList) return;
|
|
14521
|
+
await ensureDenyListData(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
14522
|
+
};
|
|
13215
14523
|
/**
|
|
13216
14524
|
* Get or build a cached search instance. Cache state
|
|
13217
|
-
* lives on the provided PipelineContext
|
|
13218
|
-
*
|
|
14525
|
+
* lives on the provided PipelineContext first. A
|
|
14526
|
+
* dictionary-scoped process cache prevents fresh
|
|
14527
|
+
* contexts with the same immutable dictionary bundle
|
|
14528
|
+
* from rebuilding identical automata.
|
|
13219
14529
|
*/
|
|
13220
14530
|
const getCachedSearch = async (config, gazetteerEntries, ctx) => {
|
|
13221
14531
|
const key = configKey(config, gazetteerEntries);
|
|
13222
14532
|
if (ctx.search && ctx.searchKey === key) return ctx.search;
|
|
13223
14533
|
if (ctx.searchPromise && ctx.searchKey === key) return ctx.searchPromise;
|
|
14534
|
+
const sharedCache = sharedSearchCacheFor(config.dictionaries);
|
|
14535
|
+
const shared = sharedCache.get(key);
|
|
14536
|
+
if (shared !== void 0) {
|
|
14537
|
+
const result = await shared;
|
|
14538
|
+
await ensureSearchSupportData(config, ctx);
|
|
14539
|
+
ctx.search = result;
|
|
14540
|
+
ctx.searchKey = key;
|
|
14541
|
+
ctx.searchPromise = null;
|
|
14542
|
+
return result;
|
|
14543
|
+
}
|
|
13224
14544
|
ctx.search = null;
|
|
13225
14545
|
ctx.searchKey = key;
|
|
13226
14546
|
const promise = buildUnifiedSearch(config, gazetteerEntries, ctx);
|
|
13227
14547
|
ctx.searchPromise = promise;
|
|
13228
|
-
|
|
14548
|
+
sharedCache.set(key, promise);
|
|
14549
|
+
let result;
|
|
14550
|
+
try {
|
|
14551
|
+
result = await promise;
|
|
14552
|
+
} catch (err) {
|
|
14553
|
+
if (sharedCache.get(key) === promise) sharedCache.delete(key);
|
|
14554
|
+
throw err;
|
|
14555
|
+
}
|
|
14556
|
+
if (sharedCache.get(key) === promise) sharedCache.set(key, result);
|
|
13229
14557
|
if (ctx.searchKey === key) ctx.search = result;
|
|
13230
14558
|
return result;
|
|
13231
14559
|
};
|
|
@@ -13278,6 +14606,7 @@ const runPipeline = async (options) => {
|
|
|
13278
14606
|
});
|
|
13279
14607
|
await Promise.all([
|
|
13280
14608
|
loadGenericRoles(ctx),
|
|
14609
|
+
loadDocumentStructureHeadings(),
|
|
13281
14610
|
initPrepositions(),
|
|
13282
14611
|
initStreetAbbrevs(),
|
|
13283
14612
|
initAddressComponents(),
|
|
@@ -13287,6 +14616,7 @@ const runPipeline = async (options) => {
|
|
|
13287
14616
|
]);
|
|
13288
14617
|
} else await Promise.all([
|
|
13289
14618
|
loadGenericRoles(ctx),
|
|
14619
|
+
loadDocumentStructureHeadings(),
|
|
13290
14620
|
initPrepositions(),
|
|
13291
14621
|
initStreetAbbrevs(),
|
|
13292
14622
|
initAddressComponents(),
|
|
@@ -13315,7 +14645,7 @@ const runPipeline = async (options) => {
|
|
|
13315
14645
|
const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
|
|
13316
14646
|
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
13317
14647
|
if (legalFormsEnabled || config.enableTriggerPhrases) await warmLegalRoleHeads();
|
|
13318
|
-
const rawLegalFormEntities = legalFormsEnabled ?
|
|
14648
|
+
const rawLegalFormEntities = legalFormsEnabled ? detectLegalFormsV2(fullText) : [];
|
|
13319
14649
|
const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
|
|
13320
14650
|
if (legalFormEntities.length > 0) log("legal-forms", `${legalFormEntities.length} matches`);
|
|
13321
14651
|
if (config.enableTriggerPhrases) await warmAddressStopKeywords();
|
|
@@ -13419,7 +14749,8 @@ const runPipeline = async (options) => {
|
|
|
13419
14749
|
}
|
|
13420
14750
|
const rawMerged = mergeAndDedup(allEntities);
|
|
13421
14751
|
log("merge", `${rawMerged.length} after dedup`);
|
|
13422
|
-
const
|
|
14752
|
+
const monetaryAmountWordsRe = await getAmountWordsRe();
|
|
14753
|
+
const mergedExtended = extendMonetaryAmountWords(extendMonetaryTrailingCurrency(rawMerged, fullText, await getMonetaryTrailingCurrencyRe()), fullText, monetaryAmountWordsRe);
|
|
13423
14754
|
const consistent = enforceBoundaryConsistency(mergedExtended, fullText);
|
|
13424
14755
|
if (consistent.length < mergedExtended.length) log("boundary", `${mergedExtended.length - consistent.length} consolidated`);
|
|
13425
14756
|
let postOrgEntities = consistent;
|