@stll/anonymize-wasm 1.4.1 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-stopwords.mjs +1 -0
- package/dist/common-words-en.mjs +9888 -0
- package/dist/common-words-en.mjs.map +1 -0
- package/dist/legal-form-leading-clauses.mjs +18 -0
- package/dist/legal-form-leading-clauses.mjs.map +1 -0
- package/dist/wasm.d.mts +43 -5
- package/dist/wasm.mjs +337 -93
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_T
|
|
|
2
2
|
import { t as amount_words_default } from "./amount-words.mjs";
|
|
3
3
|
import { t as address_street_types_default } from "./address-street-types.mjs";
|
|
4
4
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
5
|
-
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
5
|
+
import { at, au, be, bg, br, ch, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
|
|
6
6
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
7
7
|
//#region src/search-engine.ts
|
|
8
8
|
let _TextSearch;
|
|
@@ -31,6 +31,7 @@ const createPipelineContext = () => ({
|
|
|
31
31
|
searchKey: "",
|
|
32
32
|
searchPromise: null,
|
|
33
33
|
nameCorpus: null,
|
|
34
|
+
nameCorpusKey: "",
|
|
34
35
|
nameCorpusPromise: null,
|
|
35
36
|
stopwords: null,
|
|
36
37
|
stopwordsPromise: null,
|
|
@@ -889,7 +890,44 @@ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùû
|
|
|
889
890
|
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
890
891
|
const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
|
|
891
892
|
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
893
|
+
const HSPACE = "(?:[^\\S\\n]|[\xA0 ])";
|
|
894
|
+
const LEGAL_LIST_BOUNDARY_RE = new RegExp(`^[,;]${HSPACE}+(?=\\p{Lu}|(?:\\p{Lu}\\.${HSPACE}?){2,})`, "u");
|
|
892
895
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
896
|
+
const EMPTY_LEADING_CLAUSE_TRIMS = {
|
|
897
|
+
phrases: [],
|
|
898
|
+
directPrefixes: []
|
|
899
|
+
};
|
|
900
|
+
let leadingClauseTrimsCache = null;
|
|
901
|
+
let leadingClauseTrimsPromise = null;
|
|
902
|
+
const loadLeadingClauseTrims = async () => {
|
|
903
|
+
if (leadingClauseTrimsCache) return leadingClauseTrimsCache;
|
|
904
|
+
if (leadingClauseTrimsPromise) return leadingClauseTrimsPromise;
|
|
905
|
+
leadingClauseTrimsPromise = (async () => {
|
|
906
|
+
let data = {};
|
|
907
|
+
try {
|
|
908
|
+
const mod = await import("./legal-form-leading-clauses.mjs");
|
|
909
|
+
data = mod.default ?? mod;
|
|
910
|
+
} catch (err) {
|
|
911
|
+
console.warn("[anonymize] legal-forms: failed to load legal-form-leading-clauses.json:", err);
|
|
912
|
+
}
|
|
913
|
+
const phrases = /* @__PURE__ */ new Set();
|
|
914
|
+
const directPrefixes = /* @__PURE__ */ new Set();
|
|
915
|
+
for (const [key, value] of Object.entries(data)) {
|
|
916
|
+
if (key.startsWith("_") || typeof value !== "object" || value === null) continue;
|
|
917
|
+
const config = value;
|
|
918
|
+
for (const phrase of config.phrases ?? []) if (typeof phrase === "string" && phrase.length > 0) phrases.add(phrase);
|
|
919
|
+
for (const prefix of config.directPrefixes ?? []) if (typeof prefix === "string" && prefix.length > 0) directPrefixes.add(prefix);
|
|
920
|
+
}
|
|
921
|
+
const result = {
|
|
922
|
+
phrases: [...phrases],
|
|
923
|
+
directPrefixes: [...directPrefixes]
|
|
924
|
+
};
|
|
925
|
+
leadingClauseTrimsCache = result;
|
|
926
|
+
return result;
|
|
927
|
+
})();
|
|
928
|
+
return leadingClauseTrimsPromise;
|
|
929
|
+
};
|
|
930
|
+
const getLeadingClauseTrimsSync = () => leadingClauseTrimsCache ?? EMPTY_LEADING_CLAUSE_TRIMS;
|
|
893
931
|
let legalRoleHeadsCache = null;
|
|
894
932
|
let legalRoleHeadsPromise = null;
|
|
895
933
|
const loadLegalRoleHeads = async () => {
|
|
@@ -917,7 +955,8 @@ const warmLegalRoleHeads = async () => {
|
|
|
917
955
|
loadSentenceVerbIndicators(),
|
|
918
956
|
loadClauseNounHeads(),
|
|
919
957
|
loadConnectorProseHeads(),
|
|
920
|
-
loadStructuralSingleCapPrefixes()
|
|
958
|
+
loadStructuralSingleCapPrefixes(),
|
|
959
|
+
loadLeadingClauseTrims()
|
|
921
960
|
]);
|
|
922
961
|
};
|
|
923
962
|
let allLegalSuffixesCache = null;
|
|
@@ -1054,13 +1093,12 @@ const loadStructuralSingleCapPrefixes = async () => {
|
|
|
1054
1093
|
return structuralSingleCapPrefixesPromise;
|
|
1055
1094
|
};
|
|
1056
1095
|
const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
|
|
1057
|
-
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g,
|
|
1096
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, `${HSPACE}+`).replace(/\\\./g, `\\.${HSPACE}?`);
|
|
1058
1097
|
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1059
1098
|
const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1060
1099
|
const buildPatternString = (forms) => {
|
|
1061
1100
|
if (forms.length === 0) return null;
|
|
1062
1101
|
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1063
|
-
const HSPACE = "[^\\S\\n]";
|
|
1064
1102
|
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
1065
1103
|
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1066
1104
|
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
|
|
@@ -1068,7 +1106,7 @@ const buildPatternString = (forms) => {
|
|
|
1068
1106
|
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1069
1107
|
const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
|
|
1070
1108
|
const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
|
|
1071
|
-
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(
|
|
1109
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}${`(?:${HSPACE}+|,${HSPACE}*)`}(?:${alt})(?![${LOWER}])`;
|
|
1072
1110
|
};
|
|
1073
1111
|
/**
|
|
1074
1112
|
* Build legal form regex pattern strings.
|
|
@@ -1098,10 +1136,17 @@ const buildLegalFormPatterns = async () => {
|
|
|
1098
1136
|
if (longPattern) patterns.push(longPattern);
|
|
1099
1137
|
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1100
1138
|
if (shortPattern) patterns.push(shortPattern);
|
|
1101
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1102
1139
|
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1103
|
-
|
|
1104
|
-
|
|
1140
|
+
const mixedNameSep = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1141
|
+
const capitalizedNoDigitPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}`;
|
|
1142
|
+
patterns.push(`${capitalizedNoDigitPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1143
|
+
const allcapMixedPrefix = `(?:${ALLCAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD}|\\d{1,4})){1,6}`;
|
|
1144
|
+
patterns.push(`${allcapMixedPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1145
|
+
const dottedLineWrapPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}\\.${HSPACE}*\\n${HSPACE}*`;
|
|
1146
|
+
patterns.push(`${dottedLineWrapPrefix}(?:${allcapAlt})(?![${LOWER}])`);
|
|
1147
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1148
|
+
patterns.push(`${allcapPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1149
|
+
patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
|
|
1105
1150
|
return patterns;
|
|
1106
1151
|
};
|
|
1107
1152
|
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
@@ -1110,13 +1155,14 @@ const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
|
1110
1155
|
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1111
1156
|
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1112
1157
|
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1113
|
-
const
|
|
1114
|
-
const
|
|
1115
|
-
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
|
|
1158
|
+
const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:${HSPACE}+|,${HSPACE}*)`, "u");
|
|
1159
|
+
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)${HSPACE}+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:${HSPACE}+|,${HSPACE}*)`, "u");
|
|
1116
1160
|
const isStructuralSingleCapMatch = (text) => {
|
|
1117
1161
|
const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
|
|
1118
1162
|
return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
|
|
1119
1163
|
};
|
|
1164
|
+
const findLastSuffixSeparator = (text) => Math.max(text.lastIndexOf(" "), text.lastIndexOf(" "), text.lastIndexOf("\xA0"), text.lastIndexOf(" "), text.lastIndexOf(","));
|
|
1165
|
+
const stripDocxSpaces = (text) => text.replace(/[ ]/g, "");
|
|
1120
1166
|
/**
|
|
1121
1167
|
* Find the word ending just before `pos` in `text`,
|
|
1122
1168
|
* skipping any whitespace (not newlines).
|
|
@@ -1180,6 +1226,59 @@ const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
|
|
|
1180
1226
|
entityText: entityText.slice(cut)
|
|
1181
1227
|
};
|
|
1182
1228
|
};
|
|
1229
|
+
const splitEmbeddedLegalFormList = (entityStart, entityText) => {
|
|
1230
|
+
const cuts = [0];
|
|
1231
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1232
|
+
const suffixClean = suffix.replace(/[.,\s]/g, "");
|
|
1233
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1234
|
+
let fromIndex = 0;
|
|
1235
|
+
while (fromIndex < entityText.length) {
|
|
1236
|
+
const suffixStart = entityText.indexOf(suffix, fromIndex);
|
|
1237
|
+
if (suffixStart === -1) break;
|
|
1238
|
+
fromIndex = suffixStart + suffix.length;
|
|
1239
|
+
const suffixEnd = suffixStart + suffix.length;
|
|
1240
|
+
if (suffixEnd >= entityText.length - 1) continue;
|
|
1241
|
+
const afterSuffix = entityText.slice(suffixEnd);
|
|
1242
|
+
const boundary = LEGAL_LIST_BOUNDARY_RE.exec(afterSuffix);
|
|
1243
|
+
if (boundary === null) continue;
|
|
1244
|
+
const nextStart = suffixEnd + boundary[0].length;
|
|
1245
|
+
const remainder = entityText.slice(nextStart);
|
|
1246
|
+
if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
|
|
1247
|
+
cuts.push(nextStart);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
const uniqueCuts = [...new Set(cuts)].toSorted((a, b) => a - b);
|
|
1251
|
+
if (uniqueCuts.length === 1) return [{
|
|
1252
|
+
entityStart,
|
|
1253
|
+
entityText
|
|
1254
|
+
}];
|
|
1255
|
+
const segments = [];
|
|
1256
|
+
for (let index = 0; index < uniqueCuts.length; index++) {
|
|
1257
|
+
const start = uniqueCuts[index];
|
|
1258
|
+
const end = uniqueCuts[index + 1] ?? entityText.length;
|
|
1259
|
+
if (start === void 0) continue;
|
|
1260
|
+
const segmentText = entityText.slice(start, end).replace(/[,\s;]+$/u, "");
|
|
1261
|
+
if (segmentText.length === 0) continue;
|
|
1262
|
+
segments.push({
|
|
1263
|
+
entityStart: entityStart + start,
|
|
1264
|
+
entityText: segmentText
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
return segments;
|
|
1268
|
+
};
|
|
1269
|
+
const hasDisallowedLineBreak = (text) => {
|
|
1270
|
+
for (const match of text.matchAll(/\n/gu)) {
|
|
1271
|
+
const index = match.index;
|
|
1272
|
+
if (index === void 0) continue;
|
|
1273
|
+
const before = text.slice(0, index);
|
|
1274
|
+
const after = text.slice(index + 1);
|
|
1275
|
+
const dottedDesignatorBefore = /\.[^\S\n]*$/u.test(before);
|
|
1276
|
+
const legalSuffixAfter = /^[^\S\n]*(?:\p{Lu}\.[^\S\n]?){1,}\p{Lu}?\.?$/u.test(after);
|
|
1277
|
+
const allCapsSuffixAfter = /^[^\S\n]*\p{Lu}{2,}\.?$/u.test(after);
|
|
1278
|
+
if (!dottedDesignatorBefore || !legalSuffixAfter && !allCapsSuffixAfter) return true;
|
|
1279
|
+
}
|
|
1280
|
+
return false;
|
|
1281
|
+
};
|
|
1183
1282
|
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
1184
1283
|
const previousWord = findWordBefore(fullText, pos);
|
|
1185
1284
|
if (!previousWord) return false;
|
|
@@ -1336,7 +1435,29 @@ const extendBackward = (fullText, matchStart, options = {}) => {
|
|
|
1336
1435
|
};
|
|
1337
1436
|
const trimLeadingClause = (text) => {
|
|
1338
1437
|
let cut = -1;
|
|
1339
|
-
|
|
1438
|
+
const trims = getLeadingClauseTrimsSync();
|
|
1439
|
+
const phraseAlternation = trims.phrases.map(escapeForRegex).join("|");
|
|
1440
|
+
if (phraseAlternation.length > 0) {
|
|
1441
|
+
const phraseRe = new RegExp(`(?:^|\\s)(?:${phraseAlternation})${HSPACE}+`, "giu");
|
|
1442
|
+
for (const match of text.matchAll(phraseRe)) cut = Math.max(cut, match.index + match[0].length);
|
|
1443
|
+
}
|
|
1444
|
+
const directPrefixAlternation = trims.directPrefixes.map(escapeForRegex).join("|");
|
|
1445
|
+
if (directPrefixAlternation.length > 0) {
|
|
1446
|
+
const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
|
|
1447
|
+
for (const match of text.matchAll(directPrefixRe)) {
|
|
1448
|
+
const words = text.slice(0, match.index).match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
|
|
1449
|
+
if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
for (const match of text.matchAll(/,/gu)) {
|
|
1453
|
+
const comma = match.index;
|
|
1454
|
+
if (comma === void 0) continue;
|
|
1455
|
+
const before = text.slice(0, comma);
|
|
1456
|
+
if (!/\d/u.test(before)) continue;
|
|
1457
|
+
const after = text.slice(comma + 1);
|
|
1458
|
+
const leadingWs = after.match(/^\s*/u)?.[0].length ?? 0;
|
|
1459
|
+
if ((after.slice(leadingWs).match(/\p{Lu}[\p{L}\p{M}\p{N}]*/gu) ?? []).length >= 3) cut = Math.max(cut, comma + 1 + leadingWs);
|
|
1460
|
+
}
|
|
1340
1461
|
if (cut <= 0) return {
|
|
1341
1462
|
offset: 0,
|
|
1342
1463
|
text
|
|
@@ -1419,7 +1540,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1419
1540
|
}
|
|
1420
1541
|
}
|
|
1421
1542
|
}
|
|
1422
|
-
if (processedText.includes("\n")) continue;
|
|
1543
|
+
if (processedText.includes("\n") && hasDisallowedLineBreak(processedText)) continue;
|
|
1423
1544
|
let entityStart = processedStart;
|
|
1424
1545
|
let entityText = processedText;
|
|
1425
1546
|
if (fullText && !trimmed) {
|
|
@@ -1429,49 +1550,54 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1429
1550
|
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1430
1551
|
}
|
|
1431
1552
|
}
|
|
1432
|
-
const
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
return {
|
|
1443
|
-
prefixEnd,
|
|
1444
|
-
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1445
|
-
};
|
|
1446
|
-
};
|
|
1447
|
-
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1448
|
-
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1449
|
-
if (isAllCapsMatch && fullText) {
|
|
1450
|
-
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1451
|
-
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1452
|
-
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1453
|
-
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1454
|
-
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1455
|
-
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1456
|
-
entityStart = match.start;
|
|
1457
|
-
entityText = text;
|
|
1458
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1459
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1553
|
+
for (const segment of splitEmbeddedLegalFormList(entityStart, entityText)) {
|
|
1554
|
+
entityStart = segment.entityStart;
|
|
1555
|
+
entityText = segment.entityText;
|
|
1556
|
+
const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
|
|
1557
|
+
entityStart = listTrim.entityStart;
|
|
1558
|
+
entityText = listTrim.entityText;
|
|
1559
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1560
|
+
if (clauseTrim.offset > 0) {
|
|
1561
|
+
entityStart += clauseTrim.offset;
|
|
1562
|
+
entityText = clauseTrim.text;
|
|
1460
1563
|
}
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1564
|
+
if (entityText.includes("\n") && hasDisallowedLineBreak(entityText)) continue;
|
|
1565
|
+
const getPrefixInfo = (value) => {
|
|
1566
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1567
|
+
return {
|
|
1568
|
+
prefixEnd,
|
|
1569
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1570
|
+
};
|
|
1571
|
+
};
|
|
1572
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1573
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1574
|
+
if (isAllCapsMatch && fullText) {
|
|
1575
|
+
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1576
|
+
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1577
|
+
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1578
|
+
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1579
|
+
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1580
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1581
|
+
entityStart = match.start;
|
|
1582
|
+
entityText = text;
|
|
1583
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1584
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1585
|
+
}
|
|
1586
|
+
} else if (isAllCapsMatch) continue;
|
|
1587
|
+
const lastSuffixSeparator = findLastSuffixSeparator(entityText);
|
|
1588
|
+
const rawSuffix = lastSuffixSeparator !== -1 ? entityText.slice(lastSuffixSeparator + 1) : "";
|
|
1589
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1590
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1591
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(stripDocxSpaces(entityText.slice(0, lastSuffixSeparator !== -1 ? lastSuffixSeparator : entityText.length)))) continue;
|
|
1592
|
+
results.push({
|
|
1593
|
+
start: entityStart,
|
|
1594
|
+
end: entityStart + entityText.length,
|
|
1595
|
+
label: "organization",
|
|
1596
|
+
text: entityText,
|
|
1597
|
+
score: .95,
|
|
1598
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1475
1601
|
}
|
|
1476
1602
|
return results;
|
|
1477
1603
|
};
|
|
@@ -1885,20 +2011,31 @@ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList
|
|
|
1885
2011
|
* with per-language first names and surnames. When
|
|
1886
2012
|
* omitted, only legacy config files are used.
|
|
1887
2013
|
*/
|
|
1888
|
-
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
1889
|
-
|
|
2014
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
2015
|
+
const languageKey = languages?.toSorted().join(",") ?? "*";
|
|
2016
|
+
if (ctx.nameCorpus && ctx.nameCorpusKey === languageKey) return Promise.resolve();
|
|
2017
|
+
if (ctx.nameCorpusPromise && ctx.nameCorpusKey === languageKey) return ctx.nameCorpusPromise;
|
|
2018
|
+
ctx.nameCorpus = null;
|
|
2019
|
+
ctx.nameCorpusKey = languageKey;
|
|
1890
2020
|
const promise = (async () => {
|
|
1891
2021
|
try {
|
|
1892
|
-
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
2022
|
+
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod, commonWordsMod] = await Promise.all([
|
|
1893
2023
|
import("./names-first.mjs"),
|
|
1894
2024
|
import("./names-surnames.mjs"),
|
|
1895
2025
|
import("./names-title-tokens.mjs"),
|
|
1896
|
-
import("./names-exclusions.mjs")
|
|
2026
|
+
import("./names-exclusions.mjs"),
|
|
2027
|
+
import("./common-words-en.mjs")
|
|
1897
2028
|
]);
|
|
1898
2029
|
const firstNames = [...legacyFirstMod.default.names];
|
|
1899
|
-
if (dictionaries?.firstNames)
|
|
2030
|
+
if (dictionaries?.firstNames) {
|
|
2031
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.firstNames) : Object.entries(dictionaries.firstNames).filter(([language]) => languages.includes(language));
|
|
2032
|
+
for (const [, names] of entries) for (const name of names) firstNames.push(name);
|
|
2033
|
+
}
|
|
1900
2034
|
const surnames = [...legacySurnameMod.default.names];
|
|
1901
|
-
if (dictionaries?.surnames)
|
|
2035
|
+
if (dictionaries?.surnames) {
|
|
2036
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.surnames) : Object.entries(dictionaries.surnames).filter(([language]) => languages.includes(language));
|
|
2037
|
+
for (const [, names] of entries) for (const name of names) surnames.push(name);
|
|
2038
|
+
}
|
|
1902
2039
|
const dedup = (arr) => {
|
|
1903
2040
|
const seen = /* @__PURE__ */ new Set();
|
|
1904
2041
|
const result = [];
|
|
@@ -1909,8 +2046,9 @@ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
|
1909
2046
|
}
|
|
1910
2047
|
return result;
|
|
1911
2048
|
};
|
|
2049
|
+
const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
|
|
1912
2050
|
const dedupFirst = dedup(firstNames);
|
|
1913
|
-
const dedupSurnames = dedup(surnames);
|
|
2051
|
+
const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
|
|
1914
2052
|
const titles = titleMod.default.tokens;
|
|
1915
2053
|
const exclusions = exclusionMod.default.words;
|
|
1916
2054
|
ctx.nameCorpus = {
|
|
@@ -2391,9 +2529,14 @@ const STDNUM_ENTRIES = [
|
|
|
2391
2529
|
toEntry(at.uid, "tax identification number", .95),
|
|
2392
2530
|
toEntry(at.tin, "tax identification number", .9),
|
|
2393
2531
|
toEntry(at.businessid, "registration number", .95),
|
|
2532
|
+
toEntry(ch.uid, "registration number", .95),
|
|
2533
|
+
toEntry(au.abn, "tax identification number", .9),
|
|
2534
|
+
toEntry(au.acn, "registration number", .9),
|
|
2394
2535
|
toEntry(be.vat, "tax identification number", .95),
|
|
2395
2536
|
toEntry(be.nn, "national identification number", .9),
|
|
2396
2537
|
toEntry(nl.vat, "tax identification number", .95),
|
|
2538
|
+
toEntry(no.orgnr, "registration number", .9),
|
|
2539
|
+
toEntry(no.mva, "tax identification number", .95),
|
|
2397
2540
|
toEntry(dk.vat, "tax identification number", .95),
|
|
2398
2541
|
toEntry(dk.cpr, "national identification number", .9),
|
|
2399
2542
|
toEntry(fi.vat, "tax identification number", .95),
|
|
@@ -2423,6 +2566,7 @@ const STDNUM_ENTRIES = [
|
|
|
2423
2566
|
toEntry(cy.vat, "tax identification number", .95),
|
|
2424
2567
|
toEntry(mt.vat, "tax identification number", .95),
|
|
2425
2568
|
toEntry(lu.vat, "tax identification number", .95),
|
|
2569
|
+
toEntry(us.ein, "tax identification number", .9),
|
|
2426
2570
|
toEntry(br.cpf, "tax identification number", .95),
|
|
2427
2571
|
toEntry(br.cnpj, "tax identification number", .95)
|
|
2428
2572
|
].filter((e) => e !== null);
|
|
@@ -2535,6 +2679,30 @@ const ES_CIF = {
|
|
|
2535
2679
|
score: .95,
|
|
2536
2680
|
validator: es.cif
|
|
2537
2681
|
};
|
|
2682
|
+
const AU_ABN_FORMATTED = {
|
|
2683
|
+
pattern: `\\b\\d{2}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
|
|
2684
|
+
label: "tax identification number",
|
|
2685
|
+
score: .95,
|
|
2686
|
+
validator: au.abn
|
|
2687
|
+
};
|
|
2688
|
+
const NO_ORGNR_FORMATTED = {
|
|
2689
|
+
pattern: `\\b\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
|
|
2690
|
+
label: "registration number",
|
|
2691
|
+
score: .9,
|
|
2692
|
+
validator: no.orgnr
|
|
2693
|
+
};
|
|
2694
|
+
const NO_MVA_FORMATTED = {
|
|
2695
|
+
pattern: "\\bNO[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?MVA\\b",
|
|
2696
|
+
label: "tax identification number",
|
|
2697
|
+
score: .95,
|
|
2698
|
+
validator: no.mva
|
|
2699
|
+
};
|
|
2700
|
+
const US_EIN_FORMATTED = {
|
|
2701
|
+
pattern: `\\b\\d{2}${DASH}\\d{7}\\b`,
|
|
2702
|
+
label: "tax identification number",
|
|
2703
|
+
score: .95,
|
|
2704
|
+
validator: us.ein
|
|
2705
|
+
};
|
|
2538
2706
|
const BR_CPF_FORMATTED = {
|
|
2539
2707
|
pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
|
|
2540
2708
|
label: "tax identification number",
|
|
@@ -2655,6 +2823,10 @@ const ALL_REGEX_DEFS = [
|
|
|
2655
2823
|
ES_DNI,
|
|
2656
2824
|
ES_NIE,
|
|
2657
2825
|
ES_CIF,
|
|
2826
|
+
AU_ABN_FORMATTED,
|
|
2827
|
+
NO_ORGNR_FORMATTED,
|
|
2828
|
+
NO_MVA_FORMATTED,
|
|
2829
|
+
US_EIN_FORMATTED,
|
|
2658
2830
|
BR_CPF_FORMATTED,
|
|
2659
2831
|
BR_CNPJ_FORMATTED,
|
|
2660
2832
|
BR_RG_WITH_SSP,
|
|
@@ -4150,6 +4322,25 @@ const loadAllowList = (ctx) => {
|
|
|
4150
4322
|
const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
4151
4323
|
/** Sync accessor — returns empty set before init. */
|
|
4152
4324
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
4325
|
+
let commonWordsPromise = null;
|
|
4326
|
+
let commonWordsCache = null;
|
|
4327
|
+
const loadCommonWords = () => {
|
|
4328
|
+
if (commonWordsCache) return Promise.resolve(commonWordsCache);
|
|
4329
|
+
if (commonWordsPromise) return commonWordsPromise;
|
|
4330
|
+
commonWordsPromise = (async () => {
|
|
4331
|
+
try {
|
|
4332
|
+
const mod = await import("./common-words-en.mjs");
|
|
4333
|
+
const set = new Set((mod.default?.words ?? []).map((word) => word.toLowerCase()));
|
|
4334
|
+
commonWordsCache = set;
|
|
4335
|
+
return set;
|
|
4336
|
+
} catch {
|
|
4337
|
+
const empty = /* @__PURE__ */ new Set();
|
|
4338
|
+
commonWordsCache = empty;
|
|
4339
|
+
return empty;
|
|
4340
|
+
}
|
|
4341
|
+
})();
|
|
4342
|
+
return commonWordsPromise;
|
|
4343
|
+
};
|
|
4153
4344
|
/**
|
|
4154
4345
|
* Curated dictionary entries that are pure dotted
|
|
4155
4346
|
* single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
|
|
@@ -4381,6 +4572,21 @@ const SENTENCE_STARTER_WORDS = new Set([
|
|
|
4381
4572
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
4382
4573
|
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
4383
4574
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
4575
|
+
const getCityEntries = (dictionaries, allowedCountries) => {
|
|
4576
|
+
const byCountry = dictionaries?.citiesByCountry;
|
|
4577
|
+
if (!byCountry) return dictionaries?.cities ?? [];
|
|
4578
|
+
const result = [];
|
|
4579
|
+
const append = (entries) => {
|
|
4580
|
+
if (!entries) return;
|
|
4581
|
+
for (const entry of entries) result.push(entry);
|
|
4582
|
+
};
|
|
4583
|
+
if (allowedCountries === null) {
|
|
4584
|
+
for (const entries of Object.values(byCountry)) append(entries);
|
|
4585
|
+
return result;
|
|
4586
|
+
}
|
|
4587
|
+
for (const country of allowedCountries) append(byCountry[country.toUpperCase()]);
|
|
4588
|
+
return result;
|
|
4589
|
+
};
|
|
4384
4590
|
/**
|
|
4385
4591
|
* Resolve which dictionaries to load based on country
|
|
4386
4592
|
* and category filters, then build the deny list data.
|
|
@@ -4392,21 +4598,24 @@ const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^
|
|
|
4392
4598
|
* Returns null if no dictionaries are provided.
|
|
4393
4599
|
*/
|
|
4394
4600
|
const buildDenyList = async (config, ctx = defaultContext) => {
|
|
4395
|
-
await initNameCorpus(ctx, config.dictionaries);
|
|
4601
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
4396
4602
|
await Promise.all([
|
|
4397
4603
|
loadStopwords(ctx),
|
|
4398
4604
|
loadAllowList(ctx),
|
|
4399
4605
|
loadPersonStopwords(ctx),
|
|
4400
4606
|
loadAddressStopwords(ctx),
|
|
4607
|
+
loadCommonWords(),
|
|
4401
4608
|
loadStreetTypeRe(),
|
|
4402
4609
|
loadGenericRoles(ctx)
|
|
4403
4610
|
]);
|
|
4611
|
+
const commonWords = await loadCommonWords();
|
|
4404
4612
|
const dictionaries = config.dictionaries;
|
|
4405
4613
|
const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
|
|
4406
|
-
const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
|
|
4407
4614
|
const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
|
|
4408
|
-
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
4409
4615
|
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
4616
|
+
const cityEntries = getCityEntries(dictionaries, allowedCountries);
|
|
4617
|
+
const hasCities = cityEntries.length > 0;
|
|
4618
|
+
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
4410
4619
|
const excluded = config.denyListExcludeCategories;
|
|
4411
4620
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
4412
4621
|
const patternList = [];
|
|
@@ -4417,8 +4626,11 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4417
4626
|
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
4418
4627
|
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
4419
4628
|
if (normalized.length === 0) return;
|
|
4420
|
-
if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
|
|
4421
4629
|
const lower = normalized.toLowerCase();
|
|
4630
|
+
if (source !== "custom-deny-list" && label !== "address") {
|
|
4631
|
+
if (SINGLE_WORD_RE.test(normalized) && commonWords.has(lower)) return;
|
|
4632
|
+
if (isShortCuratedNoiseAcronym(normalized)) return;
|
|
4633
|
+
}
|
|
4422
4634
|
const existing = patternIndex.get(lower);
|
|
4423
4635
|
if (existing !== void 0) {
|
|
4424
4636
|
if (!labelList[existing].includes(label)) labelList[existing].push(label);
|
|
@@ -4435,10 +4647,12 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4435
4647
|
if (hasDenyList) {
|
|
4436
4648
|
const denyListData = dictionaries.denyList;
|
|
4437
4649
|
const metaData = dictionaries.denyListMeta;
|
|
4650
|
+
const useScopedNameCorpus = config.nameCorpusLanguages !== void 0;
|
|
4438
4651
|
for (const [id, entries] of Object.entries(denyListData)) {
|
|
4439
4652
|
const meta = metaData[id];
|
|
4440
4653
|
if (!meta) continue;
|
|
4441
4654
|
if (!config.enableNameCorpus && meta.category === "Names") continue;
|
|
4655
|
+
if (useScopedNameCorpus && meta.category === "Names") continue;
|
|
4442
4656
|
if (excludeCategories.has(meta.category)) continue;
|
|
4443
4657
|
if (allowedCountries !== null && meta.country !== null) {
|
|
4444
4658
|
if (!allowedCountries.has(meta.country)) continue;
|
|
@@ -4446,7 +4660,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4446
4660
|
for (const entry of entries) addDenyListEntry(entry, meta.label);
|
|
4447
4661
|
}
|
|
4448
4662
|
}
|
|
4449
|
-
if (hasCities && !excludeCategories.has("Places")) for (const entry of
|
|
4663
|
+
if (hasCities && !excludeCategories.has("Places")) for (const entry of cityEntries) addDenyListEntry(entry, "address", "city");
|
|
4450
4664
|
if (hasCustomDenyList) for (const entry of config.customDenyList) {
|
|
4451
4665
|
addDenyListEntry(entry.value, entry.label, "custom-deny-list");
|
|
4452
4666
|
for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
|
|
@@ -4541,8 +4755,8 @@ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
|
|
|
4541
4755
|
* the search instance was built on a different context
|
|
4542
4756
|
* (e.g. cachedSearch).
|
|
4543
4757
|
*/
|
|
4544
|
-
const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
4545
|
-
await initNameCorpus(ctx, dictionaries);
|
|
4758
|
+
const ensureDenyListData = async (ctx = defaultContext, dictionaries, nameCorpusLanguages) => {
|
|
4759
|
+
await initNameCorpus(ctx, dictionaries, nameCorpusLanguages);
|
|
4546
4760
|
await Promise.all([
|
|
4547
4761
|
loadStopwords(ctx),
|
|
4548
4762
|
loadAllowList(ctx),
|
|
@@ -4721,7 +4935,7 @@ const extendCityDistricts = (entities, fullText) => {
|
|
|
4721
4935
|
entity.text = fullText.slice(entity.start, entity.end);
|
|
4722
4936
|
}
|
|
4723
4937
|
const afterDistrict = fullText.slice(entity.end);
|
|
4724
|
-
const dashDistrictM = /^[\
|
|
4938
|
+
const dashDistrictM = /^[ \t]{1,4}[-–][ \t]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
|
|
4725
4939
|
if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
|
|
4726
4940
|
entity.end += dashDistrictM[0].length;
|
|
4727
4941
|
entity.text = fullText.slice(entity.start, entity.end);
|
|
@@ -4962,6 +5176,7 @@ const hasHouseNumberNearStreetWord = (fullText, seed) => {
|
|
|
4962
5176
|
const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
|
|
4963
5177
|
return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
|
|
4964
5178
|
};
|
|
5179
|
+
const isLowercaseStreetWordInProse = (fullText, seed) => /^\p{Ll}/u.test(seed.text) && /^\s+\p{Ll}/u.test(fullText.slice(seed.end, seed.end + 16)) && !hasHouseNumberNearStreetWord(fullText, seed);
|
|
4965
5180
|
const getUsZipPlusFourContext = (fullText, start, seeds) => {
|
|
4966
5181
|
const stateSeed = getUsStateSeedBeforeZip(fullText, start);
|
|
4967
5182
|
if (stateSeed !== null) return {
|
|
@@ -5022,12 +5237,14 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
5022
5237
|
for (const match of allMatches) {
|
|
5023
5238
|
const idx = match.pattern;
|
|
5024
5239
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
5025
|
-
|
|
5240
|
+
const seed = {
|
|
5026
5241
|
type: "street-word",
|
|
5027
5242
|
start: match.start,
|
|
5028
5243
|
end: match.end,
|
|
5029
5244
|
text: match.text
|
|
5030
|
-
}
|
|
5245
|
+
};
|
|
5246
|
+
if (isLowercaseStreetWordInProse(fullText, seed)) continue;
|
|
5247
|
+
seeds.push(seed);
|
|
5031
5248
|
}
|
|
5032
5249
|
for (const e of existingEntities) {
|
|
5033
5250
|
if (e.label !== "address") continue;
|
|
@@ -5166,6 +5383,7 @@ const NON_ADDRESS_LABELS = new Set([
|
|
|
5166
5383
|
]);
|
|
5167
5384
|
const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
5168
5385
|
const { start, end } = cluster;
|
|
5386
|
+
const seedTypes = new Set(cluster.seeds.map((seed) => seed.type));
|
|
5169
5387
|
let leftBound = 0;
|
|
5170
5388
|
for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
|
|
5171
5389
|
let leftPos = start;
|
|
@@ -5180,6 +5398,10 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
5180
5398
|
if (fullText.slice(p + 1, leftPos).includes("\n")) break;
|
|
5181
5399
|
leftPos = p + 1;
|
|
5182
5400
|
}
|
|
5401
|
+
if (!(seedTypes.has("street-word") || seedTypes.has("house-number") || seedTypes.has("postal-code") || seedTypes.has("address-trigger"))) return {
|
|
5402
|
+
start: Math.min(leftPos, start),
|
|
5403
|
+
end
|
|
5404
|
+
};
|
|
5183
5405
|
let rightPos = end;
|
|
5184
5406
|
const remaining = fullText.slice(rightPos);
|
|
5185
5407
|
let nearestBoundary = Math.min(remaining.length, 200);
|
|
@@ -6246,9 +6468,12 @@ const enforceBoundaryConsistency = (entities, fullText) => {
|
|
|
6246
6468
|
//#region src/build-unified-search.ts
|
|
6247
6469
|
const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
|
|
6248
6470
|
const ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
6471
|
+
const createAllowedLabelSet$1 = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
6472
|
+
const labelIsAllowed$1 = (label, allowedLabels) => allowedLabels === null || allowedLabels.has(label);
|
|
6249
6473
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
6250
6474
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
6251
|
-
const
|
|
6475
|
+
const allowedLabels = createAllowedLabelSet$1(config.enableHotwordRules === true ? expandLabelsForHotwordRules(config.labels) : config.labels);
|
|
6476
|
+
const customRegexes = config.enableRegex ? (config.customRegexes ?? []).filter((entry) => labelIsAllowed$1(entry.label, allowedLabels)) : [];
|
|
6252
6477
|
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
6253
6478
|
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
6254
6479
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
@@ -6257,22 +6482,30 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
6257
6482
|
}),
|
|
6258
6483
|
config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
|
|
6259
6484
|
buildStreetTypePatterns(),
|
|
6260
|
-
getCurrencyPatterns(),
|
|
6261
|
-
getDatePatterns(),
|
|
6262
|
-
getSigningClausePatterns()
|
|
6485
|
+
config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ? getCurrencyPatterns() : Promise.resolve([]),
|
|
6486
|
+
config.enableRegex && labelIsAllowed$1("date", allowedLabels) ? getDatePatterns() : Promise.resolve([]),
|
|
6487
|
+
config.enableRegex && labelIsAllowed$1("address", allowedLabels) ? getSigningClausePatterns() : Promise.resolve([])
|
|
6263
6488
|
]);
|
|
6264
|
-
const allRegex = [
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6489
|
+
const allRegex = [];
|
|
6490
|
+
const regexMeta = [];
|
|
6491
|
+
if (config.enableRegex) for (const [index, pattern] of REGEX_PATTERNS.entries()) {
|
|
6492
|
+
const meta = REGEX_META[index];
|
|
6493
|
+
if (!meta || !labelIsAllowed$1(meta.label, allowedLabels)) continue;
|
|
6494
|
+
allRegex.push(pattern);
|
|
6495
|
+
regexMeta.push(meta);
|
|
6496
|
+
}
|
|
6497
|
+
for (const pattern of currencyPatterns) {
|
|
6498
|
+
allRegex.push(pattern);
|
|
6499
|
+
regexMeta.push(CURRENCY_PATTERN_META);
|
|
6500
|
+
}
|
|
6501
|
+
for (const pattern of datePatterns) {
|
|
6502
|
+
allRegex.push(pattern);
|
|
6503
|
+
regexMeta.push(DATE_PATTERN_META);
|
|
6504
|
+
}
|
|
6505
|
+
for (const pattern of signingPatterns) {
|
|
6506
|
+
allRegex.push(pattern);
|
|
6507
|
+
regexMeta.push(SIGNING_CLAUSE_META);
|
|
6508
|
+
}
|
|
6276
6509
|
const customRegexMeta = customRegexes.map((entry) => ({
|
|
6277
6510
|
label: entry.label,
|
|
6278
6511
|
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
|
|
@@ -6764,7 +6997,7 @@ const configKey = (config, gazetteerEntries) => {
|
|
|
6764
6997
|
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
|
|
6765
6998
|
})).sort().join("\n") : "";
|
|
6766
6999
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
6767
|
-
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
7000
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.nameCorpusLanguages?.toSorted().join(",") ?? ""}:${config.enableRegex}:${config.labels.toSorted().join(",")}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
6768
7001
|
};
|
|
6769
7002
|
/**
|
|
6770
7003
|
* Get or build a cached search instance. Cache state
|
|
@@ -6784,6 +7017,13 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
|
|
|
6784
7017
|
return result;
|
|
6785
7018
|
};
|
|
6786
7019
|
/**
|
|
7020
|
+
* Pre-build and cache the unified search instance for a
|
|
7021
|
+
* pipeline configuration. Use the same context in
|
|
7022
|
+
* `runPipeline` to reuse the prepared automata without
|
|
7023
|
+
* passing `cachedSearch` around manually.
|
|
7024
|
+
*/
|
|
7025
|
+
const preparePipelineSearch = ({ config, gazetteerEntries = [], context }) => getCachedSearch(config, gazetteerEntries, context ?? defaultContext);
|
|
7026
|
+
/**
|
|
6787
7027
|
* Run the full detection pipeline.
|
|
6788
7028
|
*
|
|
6789
7029
|
* Two TextSearch instances scan the text (regex +
|
|
@@ -6840,7 +7080,7 @@ const runPipeline = async (options) => {
|
|
|
6840
7080
|
warmAddressStopKeywords(),
|
|
6841
7081
|
hotwordInit
|
|
6842
7082
|
]);
|
|
6843
|
-
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
7083
|
+
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
6844
7084
|
let zones = [];
|
|
6845
7085
|
if (config.enableZoneClassification && zoneInitOk) {
|
|
6846
7086
|
zones = classifyZones(fullText, ctx);
|
|
@@ -6849,7 +7089,11 @@ const runPipeline = async (options) => {
|
|
|
6849
7089
|
checkAbort(signal);
|
|
6850
7090
|
const hotwordsActive = enableHotwords && hotwordInitOk;
|
|
6851
7091
|
const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
|
|
6852
|
-
const
|
|
7092
|
+
const searchConfig = hotwordsActive ? {
|
|
7093
|
+
...config,
|
|
7094
|
+
labels: [...expandLabelsForHotwordRules(config.labels)]
|
|
7095
|
+
} : config;
|
|
7096
|
+
const search = cachedSearch ?? await getCachedSearch(searchConfig, gazetteerEntries, ctx);
|
|
6853
7097
|
checkAbort(signal);
|
|
6854
7098
|
const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
6855
7099
|
const { slices } = search;
|
|
@@ -6869,7 +7113,7 @@ const runPipeline = async (options) => {
|
|
|
6869
7113
|
let rawNameCorpusEntities = [];
|
|
6870
7114
|
let nameCorpusEntities = [];
|
|
6871
7115
|
if (config.enableNameCorpus && !config.enableDenyList) {
|
|
6872
|
-
await initNameCorpus(ctx, config.dictionaries);
|
|
7116
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
6873
7117
|
checkAbort(signal);
|
|
6874
7118
|
rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
|
|
6875
7119
|
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|
|
@@ -7568,6 +7812,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
7568
7812
|
//#region src/wasm.ts
|
|
7569
7813
|
initTextSearch(TextSearch);
|
|
7570
7814
|
//#endregion
|
|
7571
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
7815
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
7572
7816
|
|
|
7573
7817
|
//# sourceMappingURL=wasm.mjs.map
|