@stll/anonymize-wasm 1.4.1 → 1.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-stopwords.mjs +1 -0
- package/dist/common-words-en.mjs +9887 -0
- package/dist/common-words-en.mjs.map +1 -0
- package/dist/legal-form-leading-clauses.mjs +23 -0
- package/dist/legal-form-leading-clauses.mjs.map +1 -0
- package/dist/person-stopwords.mjs +6 -0
- package/dist/wasm.d.mts +43 -5
- package/dist/wasm.mjs +579 -102
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_T
|
|
|
2
2
|
import { t as amount_words_default } from "./amount-words.mjs";
|
|
3
3
|
import { t as address_street_types_default } from "./address-street-types.mjs";
|
|
4
4
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
5
|
-
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
5
|
+
import { at, au, be, bg, br, ch, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
|
|
6
6
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
7
7
|
//#region src/search-engine.ts
|
|
8
8
|
let _TextSearch;
|
|
@@ -31,6 +31,7 @@ const createPipelineContext = () => ({
|
|
|
31
31
|
searchKey: "",
|
|
32
32
|
searchPromise: null,
|
|
33
33
|
nameCorpus: null,
|
|
34
|
+
nameCorpusKey: "",
|
|
34
35
|
nameCorpusPromise: null,
|
|
35
36
|
stopwords: null,
|
|
36
37
|
stopwordsPromise: null,
|
|
@@ -230,12 +231,22 @@ const LEGAL_SUFFIXES = [...[
|
|
|
230
231
|
"OHG",
|
|
231
232
|
"Ltd.",
|
|
232
233
|
"Ltd",
|
|
234
|
+
"LTD.",
|
|
235
|
+
"LTD",
|
|
233
236
|
"LLC",
|
|
234
237
|
"LLP",
|
|
235
238
|
"Inc.",
|
|
239
|
+
"INC.",
|
|
240
|
+
"Inc",
|
|
241
|
+
"INC",
|
|
236
242
|
"Corp.",
|
|
243
|
+
"CORP.",
|
|
244
|
+
"Corp",
|
|
245
|
+
"CORP",
|
|
237
246
|
"Corporation",
|
|
247
|
+
"CORPORATION",
|
|
238
248
|
"Co.",
|
|
249
|
+
"CO.",
|
|
239
250
|
"LP",
|
|
240
251
|
"L.P.",
|
|
241
252
|
"PLC",
|
|
@@ -245,6 +256,8 @@ const LEGAL_SUFFIXES = [...[
|
|
|
245
256
|
"B.V.",
|
|
246
257
|
"Pty Ltd.",
|
|
247
258
|
"Pty Ltd",
|
|
259
|
+
"PTY LTD.",
|
|
260
|
+
"PTY LTD",
|
|
248
261
|
"S.A.",
|
|
249
262
|
"SA",
|
|
250
263
|
"SAS",
|
|
@@ -889,7 +902,44 @@ const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùû
|
|
|
889
902
|
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
890
903
|
const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
|
|
891
904
|
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
905
|
+
const HSPACE = "(?:[^\\S\\n]|[\xA0 ])";
|
|
906
|
+
const LEGAL_LIST_BOUNDARY_RE = new RegExp(`^[,;]${HSPACE}+(?=\\p{Lu}|(?:\\p{Lu}\\.${HSPACE}?){2,})`, "u");
|
|
892
907
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
908
|
+
const EMPTY_LEADING_CLAUSE_TRIMS = {
|
|
909
|
+
phrases: [],
|
|
910
|
+
directPrefixes: []
|
|
911
|
+
};
|
|
912
|
+
let leadingClauseTrimsCache = null;
|
|
913
|
+
let leadingClauseTrimsPromise = null;
|
|
914
|
+
const loadLeadingClauseTrims = async () => {
|
|
915
|
+
if (leadingClauseTrimsCache) return leadingClauseTrimsCache;
|
|
916
|
+
if (leadingClauseTrimsPromise) return leadingClauseTrimsPromise;
|
|
917
|
+
leadingClauseTrimsPromise = (async () => {
|
|
918
|
+
let data = {};
|
|
919
|
+
try {
|
|
920
|
+
const mod = await import("./legal-form-leading-clauses.mjs");
|
|
921
|
+
data = mod.default ?? mod;
|
|
922
|
+
} catch (err) {
|
|
923
|
+
console.warn("[anonymize] legal-forms: failed to load legal-form-leading-clauses.json:", err);
|
|
924
|
+
}
|
|
925
|
+
const phrases = /* @__PURE__ */ new Set();
|
|
926
|
+
const directPrefixes = /* @__PURE__ */ new Set();
|
|
927
|
+
for (const [key, value] of Object.entries(data)) {
|
|
928
|
+
if (key.startsWith("_") || typeof value !== "object" || value === null) continue;
|
|
929
|
+
const config = value;
|
|
930
|
+
for (const phrase of config.phrases ?? []) if (typeof phrase === "string" && phrase.length > 0) phrases.add(phrase);
|
|
931
|
+
for (const prefix of config.directPrefixes ?? []) if (typeof prefix === "string" && prefix.length > 0) directPrefixes.add(prefix);
|
|
932
|
+
}
|
|
933
|
+
const result = {
|
|
934
|
+
phrases: [...phrases],
|
|
935
|
+
directPrefixes: [...directPrefixes]
|
|
936
|
+
};
|
|
937
|
+
leadingClauseTrimsCache = result;
|
|
938
|
+
return result;
|
|
939
|
+
})();
|
|
940
|
+
return leadingClauseTrimsPromise;
|
|
941
|
+
};
|
|
942
|
+
const getLeadingClauseTrimsSync = () => leadingClauseTrimsCache ?? EMPTY_LEADING_CLAUSE_TRIMS;
|
|
893
943
|
let legalRoleHeadsCache = null;
|
|
894
944
|
let legalRoleHeadsPromise = null;
|
|
895
945
|
const loadLegalRoleHeads = async () => {
|
|
@@ -917,7 +967,8 @@ const warmLegalRoleHeads = async () => {
|
|
|
917
967
|
loadSentenceVerbIndicators(),
|
|
918
968
|
loadClauseNounHeads(),
|
|
919
969
|
loadConnectorProseHeads(),
|
|
920
|
-
loadStructuralSingleCapPrefixes()
|
|
970
|
+
loadStructuralSingleCapPrefixes(),
|
|
971
|
+
loadLeadingClauseTrims()
|
|
921
972
|
]);
|
|
922
973
|
};
|
|
923
974
|
let allLegalSuffixesCache = null;
|
|
@@ -1054,13 +1105,12 @@ const loadStructuralSingleCapPrefixes = async () => {
|
|
|
1054
1105
|
return structuralSingleCapPrefixesPromise;
|
|
1055
1106
|
};
|
|
1056
1107
|
const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
|
|
1057
|
-
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g,
|
|
1108
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, `${HSPACE}+`).replace(/\\\./g, `\\.${HSPACE}?`);
|
|
1058
1109
|
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1059
1110
|
const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1060
1111
|
const buildPatternString = (forms) => {
|
|
1061
1112
|
if (forms.length === 0) return null;
|
|
1062
1113
|
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1063
|
-
const HSPACE = "[^\\S\\n]";
|
|
1064
1114
|
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
1065
1115
|
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1066
1116
|
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
|
|
@@ -1068,7 +1118,7 @@ const buildPatternString = (forms) => {
|
|
|
1068
1118
|
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1069
1119
|
const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
|
|
1070
1120
|
const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
|
|
1071
|
-
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(
|
|
1121
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}${`(?:${HSPACE}+|,${HSPACE}*)`}(?:${alt})(?![${LOWER}])`;
|
|
1072
1122
|
};
|
|
1073
1123
|
/**
|
|
1074
1124
|
* Build legal form regex pattern strings.
|
|
@@ -1098,10 +1148,17 @@ const buildLegalFormPatterns = async () => {
|
|
|
1098
1148
|
if (longPattern) patterns.push(longPattern);
|
|
1099
1149
|
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1100
1150
|
if (shortPattern) patterns.push(shortPattern);
|
|
1101
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1102
1151
|
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1103
|
-
|
|
1104
|
-
|
|
1152
|
+
const mixedNameSep = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1153
|
+
const capitalizedNoDigitPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}`;
|
|
1154
|
+
patterns.push(`${capitalizedNoDigitPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1155
|
+
const allcapMixedPrefix = `(?:${ALLCAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD}|\\d{1,4})){1,6}`;
|
|
1156
|
+
patterns.push(`${allcapMixedPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1157
|
+
const dottedLineWrapPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}\\.${HSPACE}*\\n${HSPACE}*`;
|
|
1158
|
+
patterns.push(`${dottedLineWrapPrefix}(?:${allcapAlt})(?![${LOWER}])`);
|
|
1159
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1160
|
+
patterns.push(`${allcapPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1161
|
+
patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
|
|
1105
1162
|
return patterns;
|
|
1106
1163
|
};
|
|
1107
1164
|
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
@@ -1110,13 +1167,14 @@ const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
|
1110
1167
|
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1111
1168
|
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1112
1169
|
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1113
|
-
const
|
|
1114
|
-
const
|
|
1115
|
-
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
|
|
1170
|
+
const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:${HSPACE}+|,${HSPACE}*)`, "u");
|
|
1171
|
+
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)${HSPACE}+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:${HSPACE}+|,${HSPACE}*)`, "u");
|
|
1116
1172
|
const isStructuralSingleCapMatch = (text) => {
|
|
1117
1173
|
const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
|
|
1118
1174
|
return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
|
|
1119
1175
|
};
|
|
1176
|
+
const findLastSuffixSeparator = (text) => Math.max(text.lastIndexOf(" "), text.lastIndexOf(" "), text.lastIndexOf("\xA0"), text.lastIndexOf(" "), text.lastIndexOf(","));
|
|
1177
|
+
const stripDocxSpaces = (text) => text.replace(/[ ]/g, "");
|
|
1120
1178
|
/**
|
|
1121
1179
|
* Find the word ending just before `pos` in `text`,
|
|
1122
1180
|
* skipping any whitespace (not newlines).
|
|
@@ -1180,6 +1238,58 @@ const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
|
|
|
1180
1238
|
entityText: entityText.slice(cut)
|
|
1181
1239
|
};
|
|
1182
1240
|
};
|
|
1241
|
+
const splitEmbeddedLegalFormList = (entityStart, entityText) => {
|
|
1242
|
+
const cuts = [0];
|
|
1243
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1244
|
+
const suffixClean = suffix.replace(/[.,\s]/g, "");
|
|
1245
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1246
|
+
let fromIndex = 0;
|
|
1247
|
+
while (fromIndex < entityText.length) {
|
|
1248
|
+
const suffixStart = entityText.indexOf(suffix, fromIndex);
|
|
1249
|
+
if (suffixStart === -1) break;
|
|
1250
|
+
fromIndex = suffixStart + suffix.length;
|
|
1251
|
+
const suffixEnd = suffixStart + suffix.length;
|
|
1252
|
+
if (suffixEnd >= entityText.length - 1) continue;
|
|
1253
|
+
const afterSuffix = entityText.slice(suffixEnd);
|
|
1254
|
+
const boundary = LEGAL_LIST_BOUNDARY_RE.exec(afterSuffix);
|
|
1255
|
+
if (boundary === null) continue;
|
|
1256
|
+
const nextStart = suffixEnd + boundary[0].length;
|
|
1257
|
+
cuts.push(nextStart);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
const uniqueCuts = [...new Set(cuts)].toSorted((a, b) => a - b);
|
|
1261
|
+
if (uniqueCuts.length === 1) return [{
|
|
1262
|
+
entityStart,
|
|
1263
|
+
entityText
|
|
1264
|
+
}];
|
|
1265
|
+
const segments = [];
|
|
1266
|
+
for (let index = 0; index < uniqueCuts.length; index++) {
|
|
1267
|
+
const start = uniqueCuts[index];
|
|
1268
|
+
const end = uniqueCuts[index + 1] ?? entityText.length;
|
|
1269
|
+
if (start === void 0) continue;
|
|
1270
|
+
const segmentText = entityText.slice(start, end).replace(/[,\s;]+$/u, "");
|
|
1271
|
+
if (segmentText.length === 0) continue;
|
|
1272
|
+
if (!getAllLegalSuffixesSync().some((form) => segmentText.endsWith(form))) continue;
|
|
1273
|
+
segments.push({
|
|
1274
|
+
entityStart: entityStart + start,
|
|
1275
|
+
entityText: segmentText
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
return segments;
|
|
1279
|
+
};
|
|
1280
|
+
const hasDisallowedLineBreak = (text) => {
|
|
1281
|
+
for (const match of text.matchAll(/\n/gu)) {
|
|
1282
|
+
const index = match.index;
|
|
1283
|
+
if (index === void 0) continue;
|
|
1284
|
+
const before = text.slice(0, index);
|
|
1285
|
+
const after = text.slice(index + 1);
|
|
1286
|
+
const dottedDesignatorBefore = /\.[^\S\n]*$/u.test(before);
|
|
1287
|
+
const legalSuffixAfter = /^[^\S\n]*(?:\p{Lu}\.[^\S\n]?){1,}\p{Lu}?\.?$/u.test(after);
|
|
1288
|
+
const allCapsSuffixAfter = /^[^\S\n]*\p{Lu}{2,}\.?$/u.test(after);
|
|
1289
|
+
if (!dottedDesignatorBefore || !legalSuffixAfter && !allCapsSuffixAfter) return true;
|
|
1290
|
+
}
|
|
1291
|
+
return false;
|
|
1292
|
+
};
|
|
1183
1293
|
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
1184
1294
|
const previousWord = findWordBefore(fullText, pos);
|
|
1185
1295
|
if (!previousWord) return false;
|
|
@@ -1336,7 +1446,37 @@ const extendBackward = (fullText, matchStart, options = {}) => {
|
|
|
1336
1446
|
};
|
|
1337
1447
|
const trimLeadingClause = (text) => {
|
|
1338
1448
|
let cut = -1;
|
|
1339
|
-
|
|
1449
|
+
const trims = getLeadingClauseTrimsSync();
|
|
1450
|
+
const phraseAlternation = trims.phrases.map(escapeForRegex).join("|");
|
|
1451
|
+
if (phraseAlternation.length > 0) {
|
|
1452
|
+
const phraseRe = new RegExp(`(?:^|\\s)(?:${phraseAlternation})${HSPACE}+`, "giu");
|
|
1453
|
+
for (const match of text.matchAll(phraseRe)) cut = Math.max(cut, match.index + match[0].length);
|
|
1454
|
+
}
|
|
1455
|
+
const directPrefixAlternation = trims.directPrefixes.map(escapeForRegex).join("|");
|
|
1456
|
+
const COMMA_GATED_DIRECT_PREFIXES = new Set([
|
|
1457
|
+
"among",
|
|
1458
|
+
"amongst",
|
|
1459
|
+
"between"
|
|
1460
|
+
]);
|
|
1461
|
+
if (directPrefixAlternation.length > 0) {
|
|
1462
|
+
const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
|
|
1463
|
+
for (const match of text.matchAll(directPrefixRe)) {
|
|
1464
|
+
const matchedPrefix = match[0].trim().toLowerCase();
|
|
1465
|
+
const before = text.slice(0, match.index);
|
|
1466
|
+
if (COMMA_GATED_DIRECT_PREFIXES.has(matchedPrefix) && !/,\s*$/u.test(before)) continue;
|
|
1467
|
+
const words = before.match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
|
|
1468
|
+
if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
for (const match of text.matchAll(/,/gu)) {
|
|
1472
|
+
const comma = match.index;
|
|
1473
|
+
if (comma === void 0) continue;
|
|
1474
|
+
const before = text.slice(0, comma);
|
|
1475
|
+
if (!/\d/u.test(before)) continue;
|
|
1476
|
+
const after = text.slice(comma + 1);
|
|
1477
|
+
const leadingWs = after.match(/^\s*/u)?.[0].length ?? 0;
|
|
1478
|
+
if ((after.slice(leadingWs).match(/\p{Lu}[\p{L}\p{M}\p{N}]*/gu) ?? []).length >= 3) cut = Math.max(cut, comma + 1 + leadingWs);
|
|
1479
|
+
}
|
|
1340
1480
|
if (cut <= 0) return {
|
|
1341
1481
|
offset: 0,
|
|
1342
1482
|
text
|
|
@@ -1419,7 +1559,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1419
1559
|
}
|
|
1420
1560
|
}
|
|
1421
1561
|
}
|
|
1422
|
-
if (processedText.includes("\n")) continue;
|
|
1562
|
+
if (processedText.includes("\n") && hasDisallowedLineBreak(processedText)) continue;
|
|
1423
1563
|
let entityStart = processedStart;
|
|
1424
1564
|
let entityText = processedText;
|
|
1425
1565
|
if (fullText && !trimmed) {
|
|
@@ -1429,49 +1569,49 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1429
1569
|
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1430
1570
|
}
|
|
1431
1571
|
}
|
|
1432
|
-
const
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
return {
|
|
1443
|
-
prefixEnd,
|
|
1444
|
-
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1445
|
-
};
|
|
1446
|
-
};
|
|
1447
|
-
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1448
|
-
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1449
|
-
if (isAllCapsMatch && fullText) {
|
|
1450
|
-
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1451
|
-
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1452
|
-
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1453
|
-
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1454
|
-
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1455
|
-
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1456
|
-
entityStart = match.start;
|
|
1457
|
-
entityText = text;
|
|
1458
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1459
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1572
|
+
for (const segment of splitEmbeddedLegalFormList(entityStart, entityText)) {
|
|
1573
|
+
entityStart = segment.entityStart;
|
|
1574
|
+
entityText = segment.entityText;
|
|
1575
|
+
const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
|
|
1576
|
+
entityStart = listTrim.entityStart;
|
|
1577
|
+
entityText = listTrim.entityText;
|
|
1578
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1579
|
+
if (clauseTrim.offset > 0) {
|
|
1580
|
+
entityStart += clauseTrim.offset;
|
|
1581
|
+
entityText = clauseTrim.text;
|
|
1460
1582
|
}
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1583
|
+
if (entityText.includes("\n") && hasDisallowedLineBreak(entityText)) continue;
|
|
1584
|
+
const getPrefixInfo = (value) => {
|
|
1585
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1586
|
+
return {
|
|
1587
|
+
prefixEnd,
|
|
1588
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1589
|
+
};
|
|
1590
|
+
};
|
|
1591
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1592
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1593
|
+
if (isAllCapsMatch && fullText) {
|
|
1594
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1595
|
+
entityStart = match.start;
|
|
1596
|
+
entityText = text;
|
|
1597
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1598
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1599
|
+
}
|
|
1600
|
+
} else if (isAllCapsMatch) continue;
|
|
1601
|
+
const lastSuffixSeparator = findLastSuffixSeparator(entityText);
|
|
1602
|
+
const rawSuffix = lastSuffixSeparator !== -1 ? entityText.slice(lastSuffixSeparator + 1) : "";
|
|
1603
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1604
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1605
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(stripDocxSpaces(entityText.slice(0, lastSuffixSeparator !== -1 ? lastSuffixSeparator : entityText.length)))) continue;
|
|
1606
|
+
results.push({
|
|
1607
|
+
start: entityStart,
|
|
1608
|
+
end: entityStart + entityText.length,
|
|
1609
|
+
label: "organization",
|
|
1610
|
+
text: entityText,
|
|
1611
|
+
score: .95,
|
|
1612
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1475
1615
|
}
|
|
1476
1616
|
return results;
|
|
1477
1617
|
};
|
|
@@ -1885,20 +2025,31 @@ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList
|
|
|
1885
2025
|
* with per-language first names and surnames. When
|
|
1886
2026
|
* omitted, only legacy config files are used.
|
|
1887
2027
|
*/
|
|
1888
|
-
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
1889
|
-
|
|
2028
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
2029
|
+
const languageKey = languages?.toSorted().join(",") ?? "*";
|
|
2030
|
+
if (ctx.nameCorpus && ctx.nameCorpusKey === languageKey) return Promise.resolve();
|
|
2031
|
+
if (ctx.nameCorpusPromise && ctx.nameCorpusKey === languageKey) return ctx.nameCorpusPromise;
|
|
2032
|
+
ctx.nameCorpus = null;
|
|
2033
|
+
ctx.nameCorpusKey = languageKey;
|
|
1890
2034
|
const promise = (async () => {
|
|
1891
2035
|
try {
|
|
1892
|
-
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
2036
|
+
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod, commonWordsMod] = await Promise.all([
|
|
1893
2037
|
import("./names-first.mjs"),
|
|
1894
2038
|
import("./names-surnames.mjs"),
|
|
1895
2039
|
import("./names-title-tokens.mjs"),
|
|
1896
|
-
import("./names-exclusions.mjs")
|
|
2040
|
+
import("./names-exclusions.mjs"),
|
|
2041
|
+
import("./common-words-en.mjs")
|
|
1897
2042
|
]);
|
|
1898
2043
|
const firstNames = [...legacyFirstMod.default.names];
|
|
1899
|
-
if (dictionaries?.firstNames)
|
|
2044
|
+
if (dictionaries?.firstNames) {
|
|
2045
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.firstNames) : Object.entries(dictionaries.firstNames).filter(([language]) => languages.includes(language));
|
|
2046
|
+
for (const [, names] of entries) for (const name of names) firstNames.push(name);
|
|
2047
|
+
}
|
|
1900
2048
|
const surnames = [...legacySurnameMod.default.names];
|
|
1901
|
-
if (dictionaries?.surnames)
|
|
2049
|
+
if (dictionaries?.surnames) {
|
|
2050
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.surnames) : Object.entries(dictionaries.surnames).filter(([language]) => languages.includes(language));
|
|
2051
|
+
for (const [, names] of entries) for (const name of names) surnames.push(name);
|
|
2052
|
+
}
|
|
1902
2053
|
const dedup = (arr) => {
|
|
1903
2054
|
const seen = /* @__PURE__ */ new Set();
|
|
1904
2055
|
const result = [];
|
|
@@ -1909,8 +2060,9 @@ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
|
1909
2060
|
}
|
|
1910
2061
|
return result;
|
|
1911
2062
|
};
|
|
2063
|
+
const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
|
|
1912
2064
|
const dedupFirst = dedup(firstNames);
|
|
1913
|
-
const dedupSurnames = dedup(surnames);
|
|
2065
|
+
const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
|
|
1914
2066
|
const titles = titleMod.default.tokens;
|
|
1915
2067
|
const exclusions = exclusionMod.default.words;
|
|
1916
2068
|
ctx.nameCorpus = {
|
|
@@ -2007,7 +2159,31 @@ const segmentWords = (fullText) => {
|
|
|
2007
2159
|
};
|
|
2008
2160
|
/** NAME or SURNAME — both represent corpus-matched tokens */
|
|
2009
2161
|
const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
|
|
2010
|
-
const
|
|
2162
|
+
const ALL_CAPS_NAME_LINE_RATIO = .9;
|
|
2163
|
+
const ALL_CAPS_NAME_LINE_MIN_LETTERS = 3;
|
|
2164
|
+
const ALL_CAPS_NAME_LINE_MAX_TOKENS = 6;
|
|
2165
|
+
const isAllCapsLineNameShaped = (fullText, start) => {
|
|
2166
|
+
const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
|
|
2167
|
+
const lineEndIdx = fullText.indexOf("\n", start);
|
|
2168
|
+
const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
|
|
2169
|
+
if (/\d/.test(line)) return false;
|
|
2170
|
+
const tokens = line.match(/\p{L}[\p{L}\p{M}'\-]*/gu) ?? [];
|
|
2171
|
+
return tokens.length > 0 && tokens.length <= ALL_CAPS_NAME_LINE_MAX_TOKENS;
|
|
2172
|
+
};
|
|
2173
|
+
const isAllCapsContextLine = (fullText, start) => {
|
|
2174
|
+
const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
|
|
2175
|
+
const lineEndIdx = fullText.indexOf("\n", start);
|
|
2176
|
+
const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
|
|
2177
|
+
let letters = 0;
|
|
2178
|
+
let upper = 0;
|
|
2179
|
+
for (const ch of line) if (/\p{L}/u.test(ch)) {
|
|
2180
|
+
letters += 1;
|
|
2181
|
+
if (ch === ch.toUpperCase() && ch !== ch.toLowerCase()) upper += 1;
|
|
2182
|
+
}
|
|
2183
|
+
if (letters < ALL_CAPS_NAME_LINE_MIN_LETTERS) return false;
|
|
2184
|
+
return upper / letters >= ALL_CAPS_NAME_LINE_RATIO;
|
|
2185
|
+
};
|
|
2186
|
+
const classifyToken = (word, corpus, fullText) => {
|
|
2011
2187
|
const { text, start, end } = word;
|
|
2012
2188
|
const lower = text.toLowerCase();
|
|
2013
2189
|
const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
|
|
@@ -2023,6 +2199,26 @@ const classifyToken = (word, corpus) => {
|
|
|
2023
2199
|
start,
|
|
2024
2200
|
end
|
|
2025
2201
|
};
|
|
2202
|
+
if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
|
|
2203
|
+
const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
|
|
2204
|
+
const before = fullText.slice(lineStart, start).trimEnd();
|
|
2205
|
+
const lastWord = /\p{L}[\p{L}\p{M}'\-]*$/u.exec(before)?.[0];
|
|
2206
|
+
if (lastWord) {
|
|
2207
|
+
const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus);
|
|
2208
|
+
if (lookup(lastWord)) return {
|
|
2209
|
+
text,
|
|
2210
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
2211
|
+
start,
|
|
2212
|
+
end
|
|
2213
|
+
};
|
|
2214
|
+
}
|
|
2215
|
+
return {
|
|
2216
|
+
text,
|
|
2217
|
+
type: TOKEN_TYPE.OTHER,
|
|
2218
|
+
start,
|
|
2219
|
+
end
|
|
2220
|
+
};
|
|
2221
|
+
}
|
|
2026
2222
|
if (corpus.excludedWords.has(lower)) return {
|
|
2027
2223
|
text,
|
|
2028
2224
|
type: TOKEN_TYPE.OTHER,
|
|
@@ -2035,12 +2231,29 @@ const classifyToken = (word, corpus) => {
|
|
|
2035
2231
|
start,
|
|
2036
2232
|
end
|
|
2037
2233
|
};
|
|
2038
|
-
if (text.length
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2234
|
+
if (text.length >= 3 && ALL_UPPER_RE.test(text)) {
|
|
2235
|
+
if (isAllCapsContextLine(fullText, start) && isAllCapsLineNameShaped(fullText, start)) {
|
|
2236
|
+
const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
|
|
2237
|
+
if (isFirstNameToken(titleCased, corpus)) return {
|
|
2238
|
+
text,
|
|
2239
|
+
type: TOKEN_TYPE.NAME,
|
|
2240
|
+
start,
|
|
2241
|
+
end
|
|
2242
|
+
};
|
|
2243
|
+
if (isSurnameToken(titleCased, corpus)) return {
|
|
2244
|
+
text,
|
|
2245
|
+
type: TOKEN_TYPE.SURNAME,
|
|
2246
|
+
start,
|
|
2247
|
+
end
|
|
2248
|
+
};
|
|
2249
|
+
}
|
|
2250
|
+
return {
|
|
2251
|
+
text,
|
|
2252
|
+
type: TOKEN_TYPE.OTHER,
|
|
2253
|
+
start,
|
|
2254
|
+
end
|
|
2255
|
+
};
|
|
2256
|
+
}
|
|
2044
2257
|
if (!UPPER_START_RE.test(text)) return {
|
|
2045
2258
|
text,
|
|
2046
2259
|
type: TOKEN_TYPE.OTHER,
|
|
@@ -2085,7 +2298,7 @@ const classifyToken = (word, corpus) => {
|
|
|
2085
2298
|
const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
2086
2299
|
const corpus = getCorpus(ctx);
|
|
2087
2300
|
if (!corpus) return [];
|
|
2088
|
-
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
|
|
2301
|
+
const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus, fullText));
|
|
2089
2302
|
const entities = [];
|
|
2090
2303
|
const consumed = /* @__PURE__ */ new Set();
|
|
2091
2304
|
for (let i = 0; i < tokens.length; i++) {
|
|
@@ -2123,6 +2336,8 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
2123
2336
|
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
2124
2337
|
else if (hasFirstName && chain.length === 1) {
|
|
2125
2338
|
if (isSentenceStart(fullText, token.start)) continue;
|
|
2339
|
+
const first = chain[0];
|
|
2340
|
+
if (first && ALL_UPPER_RE.test(first.text) && first.text.length >= 3) continue;
|
|
2126
2341
|
score = .5;
|
|
2127
2342
|
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
2128
2343
|
else if (hasTitle && chain.length === 1) continue;
|
|
@@ -2149,6 +2364,115 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
2149
2364
|
return entities;
|
|
2150
2365
|
};
|
|
2151
2366
|
//#endregion
|
|
2367
|
+
//#region src/detectors/signatures.ts
|
|
2368
|
+
const SLASH_S_RE = /\/s\/[ \t]*/g;
|
|
2369
|
+
const LABELLED_NAME_RE = /\b(?:By|Name)[ \t]*:[ \t]*(?:\/s\/[ \t]+)?([^\n]*?)(?=\s{3,}|[\t]|\n|$)/gi;
|
|
2370
|
+
const WITNESS_ANCHOR_RE = /\bIN WITNESS WHEREOF\b[^\n]*\n/gi;
|
|
2371
|
+
const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
|
|
2372
|
+
const CAP_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'\\-]{0,30}";
|
|
2373
|
+
const NAME_SHAPE_RE = new RegExp(`^${CAP_TOKEN}(?:[ \\t]+(?:${NAME_PARTICLE}|${CAP_TOKEN})){1,4}$`, "u");
|
|
2374
|
+
const MAX_NAME_LEN = 60;
|
|
2375
|
+
const POST_NOMINAL_SUFFIX_RE = /,\s*(?:Jr|Sr|II|III|IV|V|Esq|Esquire|M\.?D|Ph\.?D|J\.?D|LL\.?M|MBA|CPA|PE|RN|DDS|DVM|DO|MD|CFA|CFP)\.?\s*$/i;
|
|
2376
|
+
const COLUMN_SEPARATOR_RE = /\s{3,}|\t+/;
|
|
2377
|
+
const ORG_SUFFIX_RE = /\b(?:INC\.?|LLC|LLP|LP|CORP\.?|CORPORATION|LTD\.?|GMBH|AG|SE|KG|OHG|SA|SAS|SARL|S\.A\.?|S\.P\.A\.?|PLC|N\.A\.?|N\.V\.?|B\.V\.?|PTY\s+LTD\.?|CO\.|S\.R\.O\.?|A\.S\.?|Z\.S\.?|S\.\s*P\.?|LTDA\.?|EIRELI|EPP|S\/A)\b/i;
|
|
2378
|
+
const IMAGE_STUB_RE = /^(?:\[?img|\[image|\[logo|\(logo\))/i;
|
|
2379
|
+
const normaliseCandidate = (text) => {
|
|
2380
|
+
let candidate = text.trim();
|
|
2381
|
+
candidate = candidate.replace(POST_NOMINAL_SUFFIX_RE, "").trim();
|
|
2382
|
+
return candidate.split(COLUMN_SEPARATOR_RE)[0]?.trim() ?? candidate;
|
|
2383
|
+
};
|
|
2384
|
+
const isNameShape = (text) => {
|
|
2385
|
+
if (text.length === 0 || text.length > MAX_NAME_LEN) return false;
|
|
2386
|
+
if (text.length < 3) return false;
|
|
2387
|
+
if (!NAME_SHAPE_RE.test(text)) return false;
|
|
2388
|
+
return true;
|
|
2389
|
+
};
|
|
2390
|
+
const findLineEnd = (text, pos) => {
|
|
2391
|
+
const idx = text.indexOf("\n", pos);
|
|
2392
|
+
return idx === -1 ? text.length : idx;
|
|
2393
|
+
};
|
|
2394
|
+
const tryEmit = (results, fullText, start, end, score) => {
|
|
2395
|
+
const raw = fullText.slice(start, end);
|
|
2396
|
+
if (ORG_SUFFIX_RE.test(raw)) return false;
|
|
2397
|
+
const candidate = normaliseCandidate(raw);
|
|
2398
|
+
if (!isNameShape(candidate)) return false;
|
|
2399
|
+
const offset = raw.indexOf(candidate);
|
|
2400
|
+
if (offset < 0) return false;
|
|
2401
|
+
const absStart = start + offset;
|
|
2402
|
+
results.push({
|
|
2403
|
+
start: absStart,
|
|
2404
|
+
end: absStart + candidate.length,
|
|
2405
|
+
label: "person",
|
|
2406
|
+
text: candidate,
|
|
2407
|
+
score,
|
|
2408
|
+
source: DETECTION_SOURCES.TRIGGER
|
|
2409
|
+
});
|
|
2410
|
+
return true;
|
|
2411
|
+
};
|
|
2412
|
+
const tryEmitForwardLines = (results, fullText, fromPos, maxLines, score) => {
|
|
2413
|
+
let pos = fromPos;
|
|
2414
|
+
for (let i = 0; i < maxLines; i++) {
|
|
2415
|
+
if (pos >= fullText.length) return false;
|
|
2416
|
+
const lineEnd = findLineEnd(fullText, pos);
|
|
2417
|
+
const line = fullText.slice(pos, lineEnd).trim();
|
|
2418
|
+
if (line.length > 0 && !IMAGE_STUB_RE.test(line)) {
|
|
2419
|
+
if (tryEmit(results, fullText, pos, lineEnd, score)) return true;
|
|
2420
|
+
}
|
|
2421
|
+
pos = lineEnd + 1;
|
|
2422
|
+
}
|
|
2423
|
+
return false;
|
|
2424
|
+
};
|
|
2425
|
+
const findPrevLine = (fullText, pos) => {
|
|
2426
|
+
let cursor = pos - 1;
|
|
2427
|
+
while (cursor >= 0 && fullText.charAt(cursor) !== "\n") cursor--;
|
|
2428
|
+
while (cursor >= 0) {
|
|
2429
|
+
let lineStart = cursor;
|
|
2430
|
+
while (lineStart > 0 && fullText.charAt(lineStart - 1) !== "\n") lineStart -= 1;
|
|
2431
|
+
const lineEnd = cursor;
|
|
2432
|
+
const line = fullText.slice(lineStart, lineEnd).trim();
|
|
2433
|
+
if (line.length > 0 && !IMAGE_STUB_RE.test(line)) return {
|
|
2434
|
+
lineStart,
|
|
2435
|
+
lineEnd
|
|
2436
|
+
};
|
|
2437
|
+
cursor = lineStart - 1;
|
|
2438
|
+
}
|
|
2439
|
+
return null;
|
|
2440
|
+
};
|
|
2441
|
+
const detectSignatures = (fullText, _ctx = defaultContext) => {
|
|
2442
|
+
const results = [];
|
|
2443
|
+
SLASH_S_RE.lastIndex = 0;
|
|
2444
|
+
for (let m = SLASH_S_RE.exec(fullText); m !== null; m = SLASH_S_RE.exec(fullText)) {
|
|
2445
|
+
const afterMark = m.index + m[0].length;
|
|
2446
|
+
const lineEnd = findLineEnd(fullText, afterMark);
|
|
2447
|
+
if (fullText.slice(afterMark, lineEnd).trim().length > 0) {
|
|
2448
|
+
const firstCell = fullText.slice(afterMark, lineEnd).split(COLUMN_SEPARATOR_RE)[0] ?? "";
|
|
2449
|
+
if (firstCell.trim().length > 0) tryEmit(results, fullText, afterMark, afterMark + firstCell.length, .95);
|
|
2450
|
+
} else tryEmitForwardLines(results, fullText, lineEnd + 1, 4, .9);
|
|
2451
|
+
const prev = findPrevLine(fullText, m.index);
|
|
2452
|
+
if (prev) tryEmit(results, fullText, prev.lineStart, prev.lineEnd, .85);
|
|
2453
|
+
}
|
|
2454
|
+
LABELLED_NAME_RE.lastIndex = 0;
|
|
2455
|
+
for (let m = LABELLED_NAME_RE.exec(fullText); m !== null; m = LABELLED_NAME_RE.exec(fullText)) {
|
|
2456
|
+
const value = m[1];
|
|
2457
|
+
if (value === void 0) continue;
|
|
2458
|
+
const valueStart = m.index + m[0].length - value.length;
|
|
2459
|
+
const valueEnd = valueStart + value.length;
|
|
2460
|
+
if (value.trim().length === 0) {
|
|
2461
|
+
tryEmitForwardLines(results, fullText, valueEnd + 1, 3, .9);
|
|
2462
|
+
continue;
|
|
2463
|
+
}
|
|
2464
|
+
tryEmit(results, fullText, valueStart, valueEnd, .95);
|
|
2465
|
+
}
|
|
2466
|
+
WITNESS_ANCHOR_RE.lastIndex = 0;
|
|
2467
|
+
for (let m = WITNESS_ANCHOR_RE.exec(fullText); m !== null; m = WITNESS_ANCHOR_RE.exec(fullText)) {
|
|
2468
|
+
const search = fullText.slice(m.index, m.index + 600);
|
|
2469
|
+
const sentenceEnd = /[.:;]\s*\n|\n\s*\n/.exec(search);
|
|
2470
|
+
if (!sentenceEnd) continue;
|
|
2471
|
+
tryEmitForwardLines(results, fullText, m.index + sentenceEnd.index + sentenceEnd[0].length, 6, .85);
|
|
2472
|
+
}
|
|
2473
|
+
return results;
|
|
2474
|
+
};
|
|
2475
|
+
//#endregion
|
|
2152
2476
|
//#region src/config/titles.ts
|
|
2153
2477
|
/**
|
|
2154
2478
|
* Academic and professional title prefixes.
|
|
@@ -2391,9 +2715,14 @@ const STDNUM_ENTRIES = [
|
|
|
2391
2715
|
toEntry(at.uid, "tax identification number", .95),
|
|
2392
2716
|
toEntry(at.tin, "tax identification number", .9),
|
|
2393
2717
|
toEntry(at.businessid, "registration number", .95),
|
|
2718
|
+
toEntry(ch.uid, "registration number", .95),
|
|
2719
|
+
toEntry(au.abn, "tax identification number", .9),
|
|
2720
|
+
toEntry(au.acn, "registration number", .9),
|
|
2394
2721
|
toEntry(be.vat, "tax identification number", .95),
|
|
2395
2722
|
toEntry(be.nn, "national identification number", .9),
|
|
2396
2723
|
toEntry(nl.vat, "tax identification number", .95),
|
|
2724
|
+
toEntry(no.orgnr, "registration number", .9),
|
|
2725
|
+
toEntry(no.mva, "tax identification number", .95),
|
|
2397
2726
|
toEntry(dk.vat, "tax identification number", .95),
|
|
2398
2727
|
toEntry(dk.cpr, "national identification number", .9),
|
|
2399
2728
|
toEntry(fi.vat, "tax identification number", .95),
|
|
@@ -2423,6 +2752,7 @@ const STDNUM_ENTRIES = [
|
|
|
2423
2752
|
toEntry(cy.vat, "tax identification number", .95),
|
|
2424
2753
|
toEntry(mt.vat, "tax identification number", .95),
|
|
2425
2754
|
toEntry(lu.vat, "tax identification number", .95),
|
|
2755
|
+
toEntry(us.ein, "tax identification number", .9),
|
|
2426
2756
|
toEntry(br.cpf, "tax identification number", .95),
|
|
2427
2757
|
toEntry(br.cnpj, "tax identification number", .95)
|
|
2428
2758
|
].filter((e) => e !== null);
|
|
@@ -2535,6 +2865,30 @@ const ES_CIF = {
|
|
|
2535
2865
|
score: .95,
|
|
2536
2866
|
validator: es.cif
|
|
2537
2867
|
};
|
|
2868
|
+
const AU_ABN_FORMATTED = {
|
|
2869
|
+
pattern: `\\b\\d{2}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
|
|
2870
|
+
label: "tax identification number",
|
|
2871
|
+
score: .95,
|
|
2872
|
+
validator: au.abn
|
|
2873
|
+
};
|
|
2874
|
+
const NO_ORGNR_FORMATTED = {
|
|
2875
|
+
pattern: `\\b\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
|
|
2876
|
+
label: "registration number",
|
|
2877
|
+
score: .9,
|
|
2878
|
+
validator: no.orgnr
|
|
2879
|
+
};
|
|
2880
|
+
const NO_MVA_FORMATTED = {
|
|
2881
|
+
pattern: "\\bNO[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?MVA\\b",
|
|
2882
|
+
label: "tax identification number",
|
|
2883
|
+
score: .95,
|
|
2884
|
+
validator: no.mva
|
|
2885
|
+
};
|
|
2886
|
+
const US_EIN_FORMATTED = {
|
|
2887
|
+
pattern: `\\b\\d{2}${DASH}\\d{7}\\b`,
|
|
2888
|
+
label: "tax identification number",
|
|
2889
|
+
score: .95,
|
|
2890
|
+
validator: us.ein
|
|
2891
|
+
};
|
|
2538
2892
|
const BR_CPF_FORMATTED = {
|
|
2539
2893
|
pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
|
|
2540
2894
|
label: "tax identification number",
|
|
@@ -2655,6 +3009,10 @@ const ALL_REGEX_DEFS = [
|
|
|
2655
3009
|
ES_DNI,
|
|
2656
3010
|
ES_NIE,
|
|
2657
3011
|
ES_CIF,
|
|
3012
|
+
AU_ABN_FORMATTED,
|
|
3013
|
+
NO_ORGNR_FORMATTED,
|
|
3014
|
+
NO_MVA_FORMATTED,
|
|
3015
|
+
US_EIN_FORMATTED,
|
|
2658
3016
|
BR_CPF_FORMATTED,
|
|
2659
3017
|
BR_CNPJ_FORMATTED,
|
|
2660
3018
|
BR_RG_WITH_SSP,
|
|
@@ -3934,6 +4292,47 @@ const MAX_ENTITY_LENGTH = {
|
|
|
3934
4292
|
organization: 80,
|
|
3935
4293
|
person: 60
|
|
3936
4294
|
};
|
|
4295
|
+
const MAX_ENTITY_WORDS = { organization: 8 };
|
|
4296
|
+
const OPEN_ENDED_SOURCES = new Set(["trigger", "coreference"]);
|
|
4297
|
+
const WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’\-./]*/gu;
|
|
4298
|
+
const countWordTokens = (text) => {
|
|
4299
|
+
let count = 0;
|
|
4300
|
+
WORD_TOKEN_RE.lastIndex = 0;
|
|
4301
|
+
while (WORD_TOKEN_RE.exec(text) !== null) count += 1;
|
|
4302
|
+
return count;
|
|
4303
|
+
};
|
|
4304
|
+
const ALL_CAPS_LINE_LETTER_THRESHOLD = 5;
|
|
4305
|
+
const ALL_CAPS_LINE_RATIO = .95;
|
|
4306
|
+
const ALL_CAPS_LINE_PROSE_EXTRA_LETTERS = 20;
|
|
4307
|
+
const ALL_CAPS_LINE_HEADING_WORD_LIMIT = 5;
|
|
4308
|
+
const SECTION_HEADING_PREFIX_RE = /^\s*(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?\s*\p{Lu}/u;
|
|
4309
|
+
const LINE_LETTER_RE = /\p{L}/gu;
|
|
4310
|
+
const ENTITY_WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’\-]*/gu;
|
|
4311
|
+
const isAllCapsBoilerplateLine = (fullText, start, length) => {
|
|
4312
|
+
const lineStart = fullText.lastIndexOf("\n", start) + 1;
|
|
4313
|
+
const lineEndIdx = fullText.indexOf("\n", start + length);
|
|
4314
|
+
const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
|
|
4315
|
+
let letterCount = 0;
|
|
4316
|
+
let upperCount = 0;
|
|
4317
|
+
let outsideEntityLetters = 0;
|
|
4318
|
+
const entityRelStart = start - lineStart;
|
|
4319
|
+
const entityRelEnd = entityRelStart + length;
|
|
4320
|
+
LINE_LETTER_RE.lastIndex = 0;
|
|
4321
|
+
for (let m = LINE_LETTER_RE.exec(line); m !== null; m = LINE_LETTER_RE.exec(line)) {
|
|
4322
|
+
const ch = m[0];
|
|
4323
|
+
letterCount += 1;
|
|
4324
|
+
if (ch === ch.toUpperCase() && ch !== ch.toLowerCase()) upperCount += 1;
|
|
4325
|
+
if (m.index < entityRelStart || m.index >= entityRelEnd) outsideEntityLetters += 1;
|
|
4326
|
+
}
|
|
4327
|
+
if (letterCount <= ALL_CAPS_LINE_LETTER_THRESHOLD) return false;
|
|
4328
|
+
if (upperCount / letterCount < ALL_CAPS_LINE_RATIO) return false;
|
|
4329
|
+
if (SECTION_HEADING_PREFIX_RE.test(line)) return true;
|
|
4330
|
+
if (outsideEntityLetters >= ALL_CAPS_LINE_PROSE_EXTRA_LETTERS) return true;
|
|
4331
|
+
const entityText = fullText.slice(start, start + length);
|
|
4332
|
+
if ((entityText.match(ENTITY_WORD_TOKEN_RE) ?? []).length > ALL_CAPS_LINE_HEADING_WORD_LIMIT && !entityText.includes(",")) return true;
|
|
4333
|
+
return false;
|
|
4334
|
+
};
|
|
4335
|
+
const isAllCapsCandidate = (text) => text === text.toUpperCase() && /\p{Lu}/u.test(text);
|
|
3937
4336
|
const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
|
|
3938
4337
|
const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
|
|
3939
4338
|
const NUMBER_ABBREV_RE = /(?:^|[\s(])(?:č|čís|nr|no|n)\.\s*$/i;
|
|
@@ -4073,6 +4472,9 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
4073
4472
|
if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
|
|
4074
4473
|
const maxLen = MAX_ENTITY_LENGTH[normalized.label];
|
|
4075
4474
|
if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
|
|
4475
|
+
const maxWords = MAX_ENTITY_WORDS[normalized.label];
|
|
4476
|
+
if (maxWords && OPEN_ENDED_SOURCES.has(normalized.source) && countWordTokens(trimmed) > maxWords) continue;
|
|
4477
|
+
if (fullText && normalized.label === "organization" && isAllCapsCandidate(trimmed) && isAllCapsBoilerplateLine(fullText, normalized.start, trimmed.length)) continue;
|
|
4076
4478
|
if (SECTION_NUMBER_RE.test(trimmed) && normalized.source !== "trigger") continue;
|
|
4077
4479
|
if (STANDALONE_YEAR_RE.test(trimmed) && normalized.source !== "trigger") continue;
|
|
4078
4480
|
if (fullText && normalized.source !== "trigger" && /^\d/.test(trimmed) && NUMBER_ABBREV_RE.test(fullText.slice(Math.max(0, normalized.start - 10), normalized.start))) continue;
|
|
@@ -4150,6 +4552,25 @@ const loadAllowList = (ctx) => {
|
|
|
4150
4552
|
const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
4151
4553
|
/** Sync accessor — returns empty set before init. */
|
|
4152
4554
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
4555
|
+
let commonWordsPromise = null;
|
|
4556
|
+
let commonWordsCache = null;
|
|
4557
|
+
const loadCommonWords = () => {
|
|
4558
|
+
if (commonWordsCache) return Promise.resolve(commonWordsCache);
|
|
4559
|
+
if (commonWordsPromise) return commonWordsPromise;
|
|
4560
|
+
commonWordsPromise = (async () => {
|
|
4561
|
+
try {
|
|
4562
|
+
const mod = await import("./common-words-en.mjs");
|
|
4563
|
+
const set = new Set((mod.default?.words ?? []).map((word) => word.toLowerCase()));
|
|
4564
|
+
commonWordsCache = set;
|
|
4565
|
+
return set;
|
|
4566
|
+
} catch {
|
|
4567
|
+
const empty = /* @__PURE__ */ new Set();
|
|
4568
|
+
commonWordsCache = empty;
|
|
4569
|
+
return empty;
|
|
4570
|
+
}
|
|
4571
|
+
})();
|
|
4572
|
+
return commonWordsPromise;
|
|
4573
|
+
};
|
|
4153
4574
|
/**
|
|
4154
4575
|
* Curated dictionary entries that are pure dotted
|
|
4155
4576
|
* single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
|
|
@@ -4381,6 +4802,21 @@ const SENTENCE_STARTER_WORDS = new Set([
|
|
|
4381
4802
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
4382
4803
|
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
4383
4804
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
4805
|
+
const getCityEntries = (dictionaries, allowedCountries) => {
|
|
4806
|
+
const byCountry = dictionaries?.citiesByCountry;
|
|
4807
|
+
if (!byCountry) return dictionaries?.cities ?? [];
|
|
4808
|
+
const result = [];
|
|
4809
|
+
const append = (entries) => {
|
|
4810
|
+
if (!entries) return;
|
|
4811
|
+
for (const entry of entries) result.push(entry);
|
|
4812
|
+
};
|
|
4813
|
+
if (allowedCountries === null) {
|
|
4814
|
+
for (const entries of Object.values(byCountry)) append(entries);
|
|
4815
|
+
return result;
|
|
4816
|
+
}
|
|
4817
|
+
for (const country of allowedCountries) append(byCountry[country.toUpperCase()]);
|
|
4818
|
+
return result;
|
|
4819
|
+
};
|
|
4384
4820
|
/**
|
|
4385
4821
|
* Resolve which dictionaries to load based on country
|
|
4386
4822
|
* and category filters, then build the deny list data.
|
|
@@ -4392,21 +4828,24 @@ const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^
|
|
|
4392
4828
|
* Returns null if no dictionaries are provided.
|
|
4393
4829
|
*/
|
|
4394
4830
|
const buildDenyList = async (config, ctx = defaultContext) => {
|
|
4395
|
-
await initNameCorpus(ctx, config.dictionaries);
|
|
4831
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
4396
4832
|
await Promise.all([
|
|
4397
4833
|
loadStopwords(ctx),
|
|
4398
4834
|
loadAllowList(ctx),
|
|
4399
4835
|
loadPersonStopwords(ctx),
|
|
4400
4836
|
loadAddressStopwords(ctx),
|
|
4837
|
+
loadCommonWords(),
|
|
4401
4838
|
loadStreetTypeRe(),
|
|
4402
4839
|
loadGenericRoles(ctx)
|
|
4403
4840
|
]);
|
|
4841
|
+
const commonWords = await loadCommonWords();
|
|
4404
4842
|
const dictionaries = config.dictionaries;
|
|
4405
4843
|
const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
|
|
4406
|
-
const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
|
|
4407
4844
|
const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
|
|
4408
|
-
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
4409
4845
|
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
4846
|
+
const cityEntries = getCityEntries(dictionaries, allowedCountries);
|
|
4847
|
+
const hasCities = cityEntries.length > 0;
|
|
4848
|
+
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
4410
4849
|
const excluded = config.denyListExcludeCategories;
|
|
4411
4850
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
4412
4851
|
const patternList = [];
|
|
@@ -4417,8 +4856,11 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4417
4856
|
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
4418
4857
|
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
4419
4858
|
if (normalized.length === 0) return;
|
|
4420
|
-
if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
|
|
4421
4859
|
const lower = normalized.toLowerCase();
|
|
4860
|
+
if (source !== "custom-deny-list" && label !== "address") {
|
|
4861
|
+
if (SINGLE_WORD_RE.test(normalized) && commonWords.has(lower)) return;
|
|
4862
|
+
if (isShortCuratedNoiseAcronym(normalized)) return;
|
|
4863
|
+
}
|
|
4422
4864
|
const existing = patternIndex.get(lower);
|
|
4423
4865
|
if (existing !== void 0) {
|
|
4424
4866
|
if (!labelList[existing].includes(label)) labelList[existing].push(label);
|
|
@@ -4435,10 +4877,12 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4435
4877
|
if (hasDenyList) {
|
|
4436
4878
|
const denyListData = dictionaries.denyList;
|
|
4437
4879
|
const metaData = dictionaries.denyListMeta;
|
|
4880
|
+
const useScopedNameCorpus = config.nameCorpusLanguages !== void 0;
|
|
4438
4881
|
for (const [id, entries] of Object.entries(denyListData)) {
|
|
4439
4882
|
const meta = metaData[id];
|
|
4440
4883
|
if (!meta) continue;
|
|
4441
4884
|
if (!config.enableNameCorpus && meta.category === "Names") continue;
|
|
4885
|
+
if (useScopedNameCorpus && meta.category === "Names") continue;
|
|
4442
4886
|
if (excludeCategories.has(meta.category)) continue;
|
|
4443
4887
|
if (allowedCountries !== null && meta.country !== null) {
|
|
4444
4888
|
if (!allowedCountries.has(meta.country)) continue;
|
|
@@ -4446,7 +4890,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4446
4890
|
for (const entry of entries) addDenyListEntry(entry, meta.label);
|
|
4447
4891
|
}
|
|
4448
4892
|
}
|
|
4449
|
-
if (hasCities && !excludeCategories.has("Places")) for (const entry of
|
|
4893
|
+
if (hasCities && !excludeCategories.has("Places")) for (const entry of cityEntries) addDenyListEntry(entry, "address", "city");
|
|
4450
4894
|
if (hasCustomDenyList) for (const entry of config.customDenyList) {
|
|
4451
4895
|
addDenyListEntry(entry.value, entry.label, "custom-deny-list");
|
|
4452
4896
|
for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
|
|
@@ -4541,8 +4985,8 @@ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
|
|
|
4541
4985
|
* the search instance was built on a different context
|
|
4542
4986
|
* (e.g. cachedSearch).
|
|
4543
4987
|
*/
|
|
4544
|
-
const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
4545
|
-
await initNameCorpus(ctx, dictionaries);
|
|
4988
|
+
const ensureDenyListData = async (ctx = defaultContext, dictionaries, nameCorpusLanguages) => {
|
|
4989
|
+
await initNameCorpus(ctx, dictionaries, nameCorpusLanguages);
|
|
4546
4990
|
await Promise.all([
|
|
4547
4991
|
loadStopwords(ctx),
|
|
4548
4992
|
loadAllowList(ctx),
|
|
@@ -4721,7 +5165,7 @@ const extendCityDistricts = (entities, fullText) => {
|
|
|
4721
5165
|
entity.text = fullText.slice(entity.start, entity.end);
|
|
4722
5166
|
}
|
|
4723
5167
|
const afterDistrict = fullText.slice(entity.end);
|
|
4724
|
-
const dashDistrictM = /^[\
|
|
5168
|
+
const dashDistrictM = /^[ \t]{1,4}[-–][ \t]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
|
|
4725
5169
|
if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
|
|
4726
5170
|
entity.end += dashDistrictM[0].length;
|
|
4727
5171
|
entity.text = fullText.slice(entity.start, entity.end);
|
|
@@ -4962,6 +5406,7 @@ const hasHouseNumberNearStreetWord = (fullText, seed) => {
|
|
|
4962
5406
|
const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
|
|
4963
5407
|
return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
|
|
4964
5408
|
};
|
|
5409
|
+
const isLowercaseStreetWordInProse = (fullText, seed) => /^\p{Ll}/u.test(seed.text) && /^\s+\p{Ll}/u.test(fullText.slice(seed.end, seed.end + 16)) && !hasHouseNumberNearStreetWord(fullText, seed);
|
|
4965
5410
|
const getUsZipPlusFourContext = (fullText, start, seeds) => {
|
|
4966
5411
|
const stateSeed = getUsStateSeedBeforeZip(fullText, start);
|
|
4967
5412
|
if (stateSeed !== null) return {
|
|
@@ -5022,12 +5467,14 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
5022
5467
|
for (const match of allMatches) {
|
|
5023
5468
|
const idx = match.pattern;
|
|
5024
5469
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
5025
|
-
|
|
5470
|
+
const seed = {
|
|
5026
5471
|
type: "street-word",
|
|
5027
5472
|
start: match.start,
|
|
5028
5473
|
end: match.end,
|
|
5029
5474
|
text: match.text
|
|
5030
|
-
}
|
|
5475
|
+
};
|
|
5476
|
+
if (isLowercaseStreetWordInProse(fullText, seed)) continue;
|
|
5477
|
+
seeds.push(seed);
|
|
5031
5478
|
}
|
|
5032
5479
|
for (const e of existingEntities) {
|
|
5033
5480
|
if (e.label !== "address") continue;
|
|
@@ -5166,6 +5613,7 @@ const NON_ADDRESS_LABELS = new Set([
|
|
|
5166
5613
|
]);
|
|
5167
5614
|
const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
5168
5615
|
const { start, end } = cluster;
|
|
5616
|
+
const seedTypes = new Set(cluster.seeds.map((seed) => seed.type));
|
|
5169
5617
|
let leftBound = 0;
|
|
5170
5618
|
for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
|
|
5171
5619
|
let leftPos = start;
|
|
@@ -5180,6 +5628,10 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
5180
5628
|
if (fullText.slice(p + 1, leftPos).includes("\n")) break;
|
|
5181
5629
|
leftPos = p + 1;
|
|
5182
5630
|
}
|
|
5631
|
+
if (!(seedTypes.has("street-word") || seedTypes.has("house-number") || seedTypes.has("postal-code") || seedTypes.has("address-trigger"))) return {
|
|
5632
|
+
start: Math.min(leftPos, start),
|
|
5633
|
+
end
|
|
5634
|
+
};
|
|
5183
5635
|
let rightPos = end;
|
|
5184
5636
|
const remaining = fullText.slice(rightPos);
|
|
5185
5637
|
let nearestBoundary = Math.min(remaining.length, 200);
|
|
@@ -6088,7 +6540,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
6088
6540
|
break;
|
|
6089
6541
|
}
|
|
6090
6542
|
}
|
|
6091
|
-
if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev)
|
|
6543
|
+
if (!hasLockedBoundary$1(prev) && !((isLegalFormOrganization(prev) || isLegalFormOrganization(entity)) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
6092
6544
|
prev.end = entity.end;
|
|
6093
6545
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
6094
6546
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -6246,9 +6698,12 @@ const enforceBoundaryConsistency = (entities, fullText) => {
|
|
|
6246
6698
|
//#region src/build-unified-search.ts
|
|
6247
6699
|
const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
|
|
6248
6700
|
const ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
6701
|
+
const createAllowedLabelSet$1 = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
6702
|
+
const labelIsAllowed$1 = (label, allowedLabels) => allowedLabels === null || allowedLabels.has(label);
|
|
6249
6703
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
6250
6704
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
6251
|
-
const
|
|
6705
|
+
const allowedLabels = createAllowedLabelSet$1(config.enableHotwordRules === true ? expandLabelsForHotwordRules(config.labels) : config.labels);
|
|
6706
|
+
const customRegexes = config.enableRegex ? (config.customRegexes ?? []).filter((entry) => labelIsAllowed$1(entry.label, allowedLabels)) : [];
|
|
6252
6707
|
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
6253
6708
|
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
6254
6709
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
@@ -6257,22 +6712,30 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
6257
6712
|
}),
|
|
6258
6713
|
config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
|
|
6259
6714
|
buildStreetTypePatterns(),
|
|
6260
|
-
getCurrencyPatterns(),
|
|
6261
|
-
getDatePatterns(),
|
|
6262
|
-
getSigningClausePatterns()
|
|
6715
|
+
config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ? getCurrencyPatterns() : Promise.resolve([]),
|
|
6716
|
+
config.enableRegex && labelIsAllowed$1("date", allowedLabels) ? getDatePatterns() : Promise.resolve([]),
|
|
6717
|
+
config.enableRegex && labelIsAllowed$1("address", allowedLabels) ? getSigningClausePatterns() : Promise.resolve([])
|
|
6263
6718
|
]);
|
|
6264
|
-
const allRegex = [
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6719
|
+
const allRegex = [];
|
|
6720
|
+
const regexMeta = [];
|
|
6721
|
+
if (config.enableRegex) for (const [index, pattern] of REGEX_PATTERNS.entries()) {
|
|
6722
|
+
const meta = REGEX_META[index];
|
|
6723
|
+
if (!meta || !labelIsAllowed$1(meta.label, allowedLabels)) continue;
|
|
6724
|
+
allRegex.push(pattern);
|
|
6725
|
+
regexMeta.push(meta);
|
|
6726
|
+
}
|
|
6727
|
+
for (const pattern of currencyPatterns) {
|
|
6728
|
+
allRegex.push(pattern);
|
|
6729
|
+
regexMeta.push(CURRENCY_PATTERN_META);
|
|
6730
|
+
}
|
|
6731
|
+
for (const pattern of datePatterns) {
|
|
6732
|
+
allRegex.push(pattern);
|
|
6733
|
+
regexMeta.push(DATE_PATTERN_META);
|
|
6734
|
+
}
|
|
6735
|
+
for (const pattern of signingPatterns) {
|
|
6736
|
+
allRegex.push(pattern);
|
|
6737
|
+
regexMeta.push(SIGNING_CLAUSE_META);
|
|
6738
|
+
}
|
|
6276
6739
|
const customRegexMeta = customRegexes.map((entry) => ({
|
|
6277
6740
|
label: entry.label,
|
|
6278
6741
|
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
|
|
@@ -6764,7 +7227,7 @@ const configKey = (config, gazetteerEntries) => {
|
|
|
6764
7227
|
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
|
|
6765
7228
|
})).sort().join("\n") : "";
|
|
6766
7229
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
6767
|
-
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
7230
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.nameCorpusLanguages?.toSorted().join(",") ?? ""}:${config.enableRegex}:${config.labels.toSorted().join(",")}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
6768
7231
|
};
|
|
6769
7232
|
/**
|
|
6770
7233
|
* Get or build a cached search instance. Cache state
|
|
@@ -6784,6 +7247,13 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
|
|
|
6784
7247
|
return result;
|
|
6785
7248
|
};
|
|
6786
7249
|
/**
|
|
7250
|
+
* Pre-build and cache the unified search instance for a
|
|
7251
|
+
* pipeline configuration. Use the same context in
|
|
7252
|
+
* `runPipeline` to reuse the prepared automata without
|
|
7253
|
+
* passing `cachedSearch` around manually.
|
|
7254
|
+
*/
|
|
7255
|
+
const preparePipelineSearch = ({ config, gazetteerEntries = [], context }) => getCachedSearch(config, gazetteerEntries, context ?? defaultContext);
|
|
7256
|
+
/**
|
|
6787
7257
|
* Run the full detection pipeline.
|
|
6788
7258
|
*
|
|
6789
7259
|
* Two TextSearch instances scan the text (regex +
|
|
@@ -6840,7 +7310,7 @@ const runPipeline = async (options) => {
|
|
|
6840
7310
|
warmAddressStopKeywords(),
|
|
6841
7311
|
hotwordInit
|
|
6842
7312
|
]);
|
|
6843
|
-
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
7313
|
+
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
6844
7314
|
let zones = [];
|
|
6845
7315
|
if (config.enableZoneClassification && zoneInitOk) {
|
|
6846
7316
|
zones = classifyZones(fullText, ctx);
|
|
@@ -6849,7 +7319,11 @@ const runPipeline = async (options) => {
|
|
|
6849
7319
|
checkAbort(signal);
|
|
6850
7320
|
const hotwordsActive = enableHotwords && hotwordInitOk;
|
|
6851
7321
|
const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
|
|
6852
|
-
const
|
|
7322
|
+
const searchConfig = hotwordsActive ? {
|
|
7323
|
+
...config,
|
|
7324
|
+
labels: [...expandLabelsForHotwordRules(config.labels)]
|
|
7325
|
+
} : config;
|
|
7326
|
+
const search = cachedSearch ?? await getCachedSearch(searchConfig, gazetteerEntries, ctx);
|
|
6853
7327
|
checkAbort(signal);
|
|
6854
7328
|
const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
6855
7329
|
const { slices } = search;
|
|
@@ -6865,11 +7339,13 @@ const runPipeline = async (options) => {
|
|
|
6865
7339
|
const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
|
|
6866
7340
|
const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
|
|
6867
7341
|
if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
|
|
7342
|
+
const signatureEntities = filterAllowedLabels(detectSignatures(fullText, ctx), preHotwordAllowedLabels);
|
|
7343
|
+
if (signatureEntities.length > 0) log("signatures", `${signatureEntities.length} matches`);
|
|
6868
7344
|
checkAbort(signal);
|
|
6869
7345
|
let rawNameCorpusEntities = [];
|
|
6870
7346
|
let nameCorpusEntities = [];
|
|
6871
7347
|
if (config.enableNameCorpus && !config.enableDenyList) {
|
|
6872
|
-
await initNameCorpus(ctx, config.dictionaries);
|
|
7348
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
6873
7349
|
checkAbort(signal);
|
|
6874
7350
|
rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
|
|
6875
7351
|
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|
|
@@ -6921,6 +7397,7 @@ const runPipeline = async (options) => {
|
|
|
6921
7397
|
checkAbort(signal);
|
|
6922
7398
|
const preAddressEntities = [
|
|
6923
7399
|
...triggerEntities,
|
|
7400
|
+
...signatureEntities,
|
|
6924
7401
|
...regexEntities,
|
|
6925
7402
|
...legalFormEntities,
|
|
6926
7403
|
...nameCorpusEntities,
|
|
@@ -7568,6 +8045,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
7568
8045
|
//#region src/wasm.ts
|
|
7569
8046
|
initTextSearch(TextSearch);
|
|
7570
8047
|
//#endregion
|
|
7571
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
8048
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
7572
8049
|
|
|
7573
8050
|
//# sourceMappingURL=wasm.mjs.map
|