@stll/anonymize-wasm 1.4.0 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-stopwords.mjs +1 -0
- package/dist/address-street-types.mjs +20 -1
- package/dist/allow-list.mjs +12 -0
- package/dist/amount-words.mjs +472 -36
- package/dist/common-words-en.mjs +9888 -0
- package/dist/common-words-en.mjs.map +1 -0
- package/dist/generic-roles.mjs +1 -0
- package/dist/legal-form-leading-clauses.mjs +18 -0
- package/dist/legal-form-leading-clauses.mjs.map +1 -0
- package/dist/rolldown-runtime.mjs +13 -0
- package/dist/structural-single-cap-prefixes.mjs +99 -0
- package/dist/structural-single-cap-prefixes.mjs.map +1 -0
- package/dist/triggers.en.mjs +16 -1
- package/dist/wasm.d.mts +54 -6
- package/dist/wasm.mjs +1040 -186
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
|
|
2
|
+
import { t as amount_words_default } from "./amount-words.mjs";
|
|
3
|
+
import { t as address_street_types_default } from "./address-street-types.mjs";
|
|
2
4
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
3
|
-
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
5
|
+
import { at, au, be, bg, br, ch, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, no, pl, pt, ro, se, si, sk, us } from "@stll/stdnum";
|
|
4
6
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
5
7
|
//#region src/search-engine.ts
|
|
6
8
|
let _TextSearch;
|
|
@@ -29,6 +31,7 @@ const createPipelineContext = () => ({
|
|
|
29
31
|
searchKey: "",
|
|
30
32
|
searchPromise: null,
|
|
31
33
|
nameCorpus: null,
|
|
34
|
+
nameCorpusKey: "",
|
|
32
35
|
nameCorpusPromise: null,
|
|
33
36
|
stopwords: null,
|
|
34
37
|
stopwordsPromise: null,
|
|
@@ -885,8 +888,46 @@ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTE
|
|
|
885
888
|
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
886
889
|
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
887
890
|
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
891
|
+
const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
|
|
888
892
|
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
893
|
+
const HSPACE = "(?:[^\\S\\n]|[\xA0 ])";
|
|
894
|
+
const LEGAL_LIST_BOUNDARY_RE = new RegExp(`^[,;]${HSPACE}+(?=\\p{Lu}|(?:\\p{Lu}\\.${HSPACE}?){2,})`, "u");
|
|
889
895
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
896
|
+
const EMPTY_LEADING_CLAUSE_TRIMS = {
|
|
897
|
+
phrases: [],
|
|
898
|
+
directPrefixes: []
|
|
899
|
+
};
|
|
900
|
+
let leadingClauseTrimsCache = null;
|
|
901
|
+
let leadingClauseTrimsPromise = null;
|
|
902
|
+
const loadLeadingClauseTrims = async () => {
|
|
903
|
+
if (leadingClauseTrimsCache) return leadingClauseTrimsCache;
|
|
904
|
+
if (leadingClauseTrimsPromise) return leadingClauseTrimsPromise;
|
|
905
|
+
leadingClauseTrimsPromise = (async () => {
|
|
906
|
+
let data = {};
|
|
907
|
+
try {
|
|
908
|
+
const mod = await import("./legal-form-leading-clauses.mjs");
|
|
909
|
+
data = mod.default ?? mod;
|
|
910
|
+
} catch (err) {
|
|
911
|
+
console.warn("[anonymize] legal-forms: failed to load legal-form-leading-clauses.json:", err);
|
|
912
|
+
}
|
|
913
|
+
const phrases = /* @__PURE__ */ new Set();
|
|
914
|
+
const directPrefixes = /* @__PURE__ */ new Set();
|
|
915
|
+
for (const [key, value] of Object.entries(data)) {
|
|
916
|
+
if (key.startsWith("_") || typeof value !== "object" || value === null) continue;
|
|
917
|
+
const config = value;
|
|
918
|
+
for (const phrase of config.phrases ?? []) if (typeof phrase === "string" && phrase.length > 0) phrases.add(phrase);
|
|
919
|
+
for (const prefix of config.directPrefixes ?? []) if (typeof prefix === "string" && prefix.length > 0) directPrefixes.add(prefix);
|
|
920
|
+
}
|
|
921
|
+
const result = {
|
|
922
|
+
phrases: [...phrases],
|
|
923
|
+
directPrefixes: [...directPrefixes]
|
|
924
|
+
};
|
|
925
|
+
leadingClauseTrimsCache = result;
|
|
926
|
+
return result;
|
|
927
|
+
})();
|
|
928
|
+
return leadingClauseTrimsPromise;
|
|
929
|
+
};
|
|
930
|
+
const getLeadingClauseTrimsSync = () => leadingClauseTrimsCache ?? EMPTY_LEADING_CLAUSE_TRIMS;
|
|
890
931
|
let legalRoleHeadsCache = null;
|
|
891
932
|
let legalRoleHeadsPromise = null;
|
|
892
933
|
const loadLegalRoleHeads = async () => {
|
|
@@ -912,11 +953,23 @@ const warmLegalRoleHeads = async () => {
|
|
|
912
953
|
loadLegalRoleHeads(),
|
|
913
954
|
loadAllLegalSuffixes(),
|
|
914
955
|
loadSentenceVerbIndicators(),
|
|
915
|
-
loadClauseNounHeads()
|
|
956
|
+
loadClauseNounHeads(),
|
|
957
|
+
loadConnectorProseHeads(),
|
|
958
|
+
loadStructuralSingleCapPrefixes(),
|
|
959
|
+
loadLeadingClauseTrims()
|
|
916
960
|
]);
|
|
917
961
|
};
|
|
918
962
|
let allLegalSuffixesCache = null;
|
|
919
963
|
let allLegalSuffixesPromise = null;
|
|
964
|
+
let normalizedLegalBoundarySuffixesCache = null;
|
|
965
|
+
let normalizedInNameLegalFormWordsCache = null;
|
|
966
|
+
const normalizeLegalSuffixToken = (suffix) => suffix.replace(/[.,\s]/g, "");
|
|
967
|
+
const isBoundaryLegalSuffixForm = (form) => {
|
|
968
|
+
const normalized = normalizeLegalSuffixToken(form);
|
|
969
|
+
if (normalized.length === 0) return false;
|
|
970
|
+
if (LEGAL_SUFFIXES.includes(form)) return true;
|
|
971
|
+
return /[.]/u.test(form) || normalized === normalized.toUpperCase();
|
|
972
|
+
};
|
|
920
973
|
const loadAllLegalSuffixes = async () => {
|
|
921
974
|
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
922
975
|
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
@@ -941,11 +994,15 @@ const loadAllLegalSuffixes = async () => {
|
|
|
941
994
|
}
|
|
942
995
|
out.sort((a, b) => b.length - a.length);
|
|
943
996
|
allLegalSuffixesCache = out;
|
|
997
|
+
normalizedLegalBoundarySuffixesCache = new Set(out.filter(isBoundaryLegalSuffixForm).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
998
|
+
normalizedInNameLegalFormWordsCache = new Set(out.filter((form) => !isBoundaryLegalSuffixForm(form) && !/\s/u.test(form)).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
944
999
|
return out;
|
|
945
1000
|
})();
|
|
946
1001
|
return allLegalSuffixesPromise;
|
|
947
1002
|
};
|
|
948
1003
|
const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
|
|
1004
|
+
const getNormalizedLegalBoundarySuffixesSync = () => normalizedLegalBoundarySuffixesCache ?? new Set(LEGAL_SUFFIXES.map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
1005
|
+
const getNormalizedInNameLegalFormWordsSync = () => normalizedInNameLegalFormWordsCache ?? /* @__PURE__ */ new Set();
|
|
949
1006
|
/**
|
|
950
1007
|
* Sync accessor for the full legal-form vocabulary
|
|
951
1008
|
* (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
|
|
@@ -985,18 +1042,71 @@ const loadClauseNounHeads = async () => {
|
|
|
985
1042
|
return clauseNounHeadsPromise;
|
|
986
1043
|
};
|
|
987
1044
|
const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
|
|
988
|
-
|
|
1045
|
+
let connectorProseHeadsCache = null;
|
|
1046
|
+
let connectorProseHeadsPromise = null;
|
|
1047
|
+
const loadConnectorProseHeads = async () => {
|
|
1048
|
+
if (connectorProseHeadsCache) return connectorProseHeadsCache;
|
|
1049
|
+
if (connectorProseHeadsPromise) return connectorProseHeadsPromise;
|
|
1050
|
+
connectorProseHeadsPromise = (async () => {
|
|
1051
|
+
let data = {};
|
|
1052
|
+
try {
|
|
1053
|
+
const mod = await import("./generic-roles.mjs");
|
|
1054
|
+
data = mod.default ?? mod;
|
|
1055
|
+
} catch (err) {
|
|
1056
|
+
console.warn("[anonymize] legal-forms: failed to load generic-roles.json:", err);
|
|
1057
|
+
}
|
|
1058
|
+
const all = /* @__PURE__ */ new Set();
|
|
1059
|
+
if (Array.isArray(data.roles)) {
|
|
1060
|
+
for (const role of data.roles) if (typeof role === "string" && role.length > 0) all.add(role.toLowerCase());
|
|
1061
|
+
}
|
|
1062
|
+
connectorProseHeadsCache = all;
|
|
1063
|
+
return all;
|
|
1064
|
+
})();
|
|
1065
|
+
return connectorProseHeadsPromise;
|
|
1066
|
+
};
|
|
1067
|
+
const getConnectorProseHeadsSync = () => connectorProseHeadsCache ?? /* @__PURE__ */ new Set();
|
|
1068
|
+
let structuralSingleCapPrefixesCache = null;
|
|
1069
|
+
let structuralSingleCapPrefixesPromise = null;
|
|
1070
|
+
const loadStructuralSingleCapPrefixes = async () => {
|
|
1071
|
+
if (structuralSingleCapPrefixesCache) return structuralSingleCapPrefixesCache;
|
|
1072
|
+
if (structuralSingleCapPrefixesPromise) return structuralSingleCapPrefixesPromise;
|
|
1073
|
+
structuralSingleCapPrefixesPromise = (async () => {
|
|
1074
|
+
let data = {};
|
|
1075
|
+
try {
|
|
1076
|
+
const mod = await import("./structural-single-cap-prefixes.mjs");
|
|
1077
|
+
data = mod.default ?? mod;
|
|
1078
|
+
} catch (err) {
|
|
1079
|
+
console.warn("[anonymize] legal-forms: failed to load structural-single-cap-prefixes.json:", err);
|
|
1080
|
+
}
|
|
1081
|
+
const all = /* @__PURE__ */ new Set();
|
|
1082
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1083
|
+
if (key.startsWith("_")) continue;
|
|
1084
|
+
if (!Array.isArray(value)) continue;
|
|
1085
|
+
for (const prefix of value) {
|
|
1086
|
+
if (typeof prefix !== "string" || prefix.length === 0) continue;
|
|
1087
|
+
all.add(prefix.toLowerCase());
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
structuralSingleCapPrefixesCache = all;
|
|
1091
|
+
return all;
|
|
1092
|
+
})();
|
|
1093
|
+
return structuralSingleCapPrefixesPromise;
|
|
1094
|
+
};
|
|
1095
|
+
const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
|
|
1096
|
+
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, `${HSPACE}+`).replace(/\\\./g, `\\.${HSPACE}?`);
|
|
989
1097
|
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1098
|
+
const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
990
1099
|
const buildPatternString = (forms) => {
|
|
991
1100
|
if (forms.length === 0) return null;
|
|
992
1101
|
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
993
|
-
const HSPACE = "[^\\S\\n]";
|
|
994
1102
|
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
995
1103
|
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
996
|
-
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
|
|
1104
|
+
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
|
|
997
1105
|
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
998
1106
|
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
999
|
-
|
|
1107
|
+
const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
|
|
1108
|
+
const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
|
|
1109
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}${`(?:${HSPACE}+|,${HSPACE}*)`}(?:${alt})(?![${LOWER}])`;
|
|
1000
1110
|
};
|
|
1001
1111
|
/**
|
|
1002
1112
|
* Build legal form regex pattern strings.
|
|
@@ -1026,9 +1136,17 @@ const buildLegalFormPatterns = async () => {
|
|
|
1026
1136
|
if (longPattern) patterns.push(longPattern);
|
|
1027
1137
|
const shortPattern = buildPatternString(allForms.filter(isShortForm));
|
|
1028
1138
|
if (shortPattern) patterns.push(shortPattern);
|
|
1029
|
-
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1030
1139
|
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1031
|
-
|
|
1140
|
+
const mixedNameSep = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1141
|
+
const capitalizedNoDigitPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}`;
|
|
1142
|
+
patterns.push(`${capitalizedNoDigitPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1143
|
+
const allcapMixedPrefix = `(?:${ALLCAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD}|\\d{1,4})){1,6}`;
|
|
1144
|
+
patterns.push(`${allcapMixedPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1145
|
+
const dottedLineWrapPrefix = `(?:${CAP_WORD})(?:${mixedNameSep}(?:${CAP_WORD}|${ALLCAP_WORD})){1,8}\\.${HSPACE}*\\n${HSPACE}*`;
|
|
1146
|
+
patterns.push(`${dottedLineWrapPrefix}(?:${allcapAlt})(?![${LOWER}])`);
|
|
1147
|
+
const allcapPrefix = `(?:${ALLCAP_WORD})(?:(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1148
|
+
patterns.push(`${allcapPrefix}(?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1149
|
+
patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:${HSPACE}+|,${HSPACE}*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
|
|
1032
1150
|
return patterns;
|
|
1033
1151
|
};
|
|
1034
1152
|
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
@@ -1037,7 +1155,14 @@ const UPPER_LETTER_RE = /^\p{Lu}/u;
|
|
|
1037
1155
|
const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Holdings|Trust|Partners|Associates|Corporation|Industries|Enterprises|Solutions|Systems|Services|Foundation|Institute)$/;
|
|
1038
1156
|
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1039
1157
|
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1040
|
-
const
|
|
1158
|
+
const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:${HSPACE}+|,${HSPACE}*)`, "u");
|
|
1159
|
+
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)${HSPACE}+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:${HSPACE}+|,${HSPACE}*)`, "u");
|
|
1160
|
+
const isStructuralSingleCapMatch = (text) => {
|
|
1161
|
+
const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
|
|
1162
|
+
return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
|
|
1163
|
+
};
|
|
1164
|
+
const findLastSuffixSeparator = (text) => Math.max(text.lastIndexOf(" "), text.lastIndexOf(" "), text.lastIndexOf("\xA0"), text.lastIndexOf(" "), text.lastIndexOf(","));
|
|
1165
|
+
const stripDocxSpaces = (text) => text.replace(/[ ]/g, "");
|
|
1041
1166
|
/**
|
|
1042
1167
|
* Find the word ending just before `pos` in `text`,
|
|
1043
1168
|
* skipping any whitespace (not newlines).
|
|
@@ -1062,25 +1187,198 @@ const findWordBefore = (text, pos) => {
|
|
|
1062
1187
|
start: wordStart
|
|
1063
1188
|
};
|
|
1064
1189
|
};
|
|
1190
|
+
const hasSingleCapPrefixBefore = (fullText, matchStart) => {
|
|
1191
|
+
const prev = findWordBefore(fullText, matchStart);
|
|
1192
|
+
return prev !== null && prev.word.length === 1 && UPPER_LETTER_RE.test(prev.word);
|
|
1193
|
+
};
|
|
1194
|
+
const isBareSingleCapStructuralInnerMatch = (fullText, matchStart, text) => {
|
|
1195
|
+
if (!BARE_SINGLE_CAP_LEGAL_FORM_RE.test(text)) return false;
|
|
1196
|
+
const prev = findWordBefore(fullText, matchStart);
|
|
1197
|
+
return prev !== null && getStructuralSingleCapPrefixesSync().has(prev.word.toLowerCase());
|
|
1198
|
+
};
|
|
1199
|
+
const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
|
|
1200
|
+
let cut = -1;
|
|
1201
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1202
|
+
const suffixClean = suffix.replace(/[.,\s]/g, "");
|
|
1203
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1204
|
+
let fromIndex = 0;
|
|
1205
|
+
while (fromIndex < entityText.length) {
|
|
1206
|
+
const suffixStart = entityText.indexOf(suffix, fromIndex);
|
|
1207
|
+
if (suffixStart === -1) break;
|
|
1208
|
+
fromIndex = suffixStart + suffix.length;
|
|
1209
|
+
const suffixEnd = suffixStart + suffix.length;
|
|
1210
|
+
if (suffixEnd >= entityText.length - 1) continue;
|
|
1211
|
+
const afterSuffix = entityText.slice(suffixEnd);
|
|
1212
|
+
const boundary = /^,\s+(?=\p{Lu})/u.exec(afterSuffix);
|
|
1213
|
+
if (boundary === null) continue;
|
|
1214
|
+
const nextStart = suffixEnd + boundary[0].length;
|
|
1215
|
+
const remainder = entityText.slice(nextStart);
|
|
1216
|
+
if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
|
|
1217
|
+
cut = Math.max(cut, nextStart);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
if (cut <= 0) return {
|
|
1221
|
+
entityStart,
|
|
1222
|
+
entityText
|
|
1223
|
+
};
|
|
1224
|
+
return {
|
|
1225
|
+
entityStart: entityStart + cut,
|
|
1226
|
+
entityText: entityText.slice(cut)
|
|
1227
|
+
};
|
|
1228
|
+
};
|
|
1229
|
+
const splitEmbeddedLegalFormList = (entityStart, entityText) => {
|
|
1230
|
+
const cuts = [0];
|
|
1231
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1232
|
+
const suffixClean = suffix.replace(/[.,\s]/g, "");
|
|
1233
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1234
|
+
let fromIndex = 0;
|
|
1235
|
+
while (fromIndex < entityText.length) {
|
|
1236
|
+
const suffixStart = entityText.indexOf(suffix, fromIndex);
|
|
1237
|
+
if (suffixStart === -1) break;
|
|
1238
|
+
fromIndex = suffixStart + suffix.length;
|
|
1239
|
+
const suffixEnd = suffixStart + suffix.length;
|
|
1240
|
+
if (suffixEnd >= entityText.length - 1) continue;
|
|
1241
|
+
const afterSuffix = entityText.slice(suffixEnd);
|
|
1242
|
+
const boundary = LEGAL_LIST_BOUNDARY_RE.exec(afterSuffix);
|
|
1243
|
+
if (boundary === null) continue;
|
|
1244
|
+
const nextStart = suffixEnd + boundary[0].length;
|
|
1245
|
+
const remainder = entityText.slice(nextStart);
|
|
1246
|
+
if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
|
|
1247
|
+
cuts.push(nextStart);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
const uniqueCuts = [...new Set(cuts)].toSorted((a, b) => a - b);
|
|
1251
|
+
if (uniqueCuts.length === 1) return [{
|
|
1252
|
+
entityStart,
|
|
1253
|
+
entityText
|
|
1254
|
+
}];
|
|
1255
|
+
const segments = [];
|
|
1256
|
+
for (let index = 0; index < uniqueCuts.length; index++) {
|
|
1257
|
+
const start = uniqueCuts[index];
|
|
1258
|
+
const end = uniqueCuts[index + 1] ?? entityText.length;
|
|
1259
|
+
if (start === void 0) continue;
|
|
1260
|
+
const segmentText = entityText.slice(start, end).replace(/[,\s;]+$/u, "");
|
|
1261
|
+
if (segmentText.length === 0) continue;
|
|
1262
|
+
segments.push({
|
|
1263
|
+
entityStart: entityStart + start,
|
|
1264
|
+
entityText: segmentText
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
return segments;
|
|
1268
|
+
};
|
|
1269
|
+
const hasDisallowedLineBreak = (text) => {
|
|
1270
|
+
for (const match of text.matchAll(/\n/gu)) {
|
|
1271
|
+
const index = match.index;
|
|
1272
|
+
if (index === void 0) continue;
|
|
1273
|
+
const before = text.slice(0, index);
|
|
1274
|
+
const after = text.slice(index + 1);
|
|
1275
|
+
const dottedDesignatorBefore = /\.[^\S\n]*$/u.test(before);
|
|
1276
|
+
const legalSuffixAfter = /^[^\S\n]*(?:\p{Lu}\.[^\S\n]?){1,}\p{Lu}?\.?$/u.test(after);
|
|
1277
|
+
const allCapsSuffixAfter = /^[^\S\n]*\p{Lu}{2,}\.?$/u.test(after);
|
|
1278
|
+
if (!dottedDesignatorBefore || !legalSuffixAfter && !allCapsSuffixAfter) return true;
|
|
1279
|
+
}
|
|
1280
|
+
return false;
|
|
1281
|
+
};
|
|
1282
|
+
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
1283
|
+
const previousWord = findWordBefore(fullText, pos);
|
|
1284
|
+
if (!previousWord) return false;
|
|
1285
|
+
let scan = previousWord.start - 1;
|
|
1286
|
+
while (scan >= 0 && (fullText[scan] === " " || fullText[scan] === " ")) scan--;
|
|
1287
|
+
return scan >= 1 && fullText[scan] === "." && UPPER_LETTER_RE.test(fullText[scan - 1] ?? "");
|
|
1288
|
+
};
|
|
1065
1289
|
/**
|
|
1066
1290
|
* Count consecutive uppercase-starting words immediately
|
|
1067
1291
|
* before `pos`. Stops at the first non-upper word or at
|
|
1068
|
-
* text/line start. Used to disambiguate
|
|
1069
|
-
* and <ORG>"
|
|
1070
|
-
|
|
1071
|
-
|
|
1292
|
+
* text/line start. Used to disambiguate sentence prose
|
|
1293
|
+
* ("<First> <Last> and <ORG>", "<Defined-Term> and
|
|
1294
|
+
* <ORG>") from multi-word organisation names that span
|
|
1295
|
+
* an "and" connector ("UniCredit Bank Czech Republic and
|
|
1296
|
+
* Slovakia, a.s.").
|
|
1297
|
+
*
|
|
1298
|
+
* When `crossInNamePreps` is true, the walk also steps
|
|
1299
|
+
* over in-name lowercase prepositions ("of", "the") as
|
|
1300
|
+
* long as they sit between two upper words. This lets
|
|
1301
|
+
* the suffix-mode "and"-crossing logic see through
|
|
1302
|
+
* "<Trust ← and ← America ← of ← Bank>" and emit one
|
|
1303
|
+
* full organisation span.
|
|
1304
|
+
*/
|
|
1305
|
+
const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
|
|
1072
1306
|
let count = 0;
|
|
1073
1307
|
let scan = pos;
|
|
1074
1308
|
while (scan > 0) {
|
|
1075
1309
|
const found = findWordBefore(fullText, scan);
|
|
1076
|
-
if (
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1310
|
+
if (found) {
|
|
1311
|
+
if (UPPER_LETTER_RE.test(found.word)) {
|
|
1312
|
+
count++;
|
|
1313
|
+
scan = found.start;
|
|
1314
|
+
continue;
|
|
1315
|
+
}
|
|
1316
|
+
if (crossInNamePreps && IN_NAME_PREPOSITION_RE.test(found.word)) {
|
|
1317
|
+
const prev = findWordBefore(fullText, found.start);
|
|
1318
|
+
if (!prev) break;
|
|
1319
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1320
|
+
scan = found.start;
|
|
1321
|
+
continue;
|
|
1322
|
+
}
|
|
1323
|
+
break;
|
|
1324
|
+
}
|
|
1325
|
+
let p = scan - 1;
|
|
1326
|
+
while (p >= 0 && (fullText[p] === " " || fullText[p] === " ")) p--;
|
|
1327
|
+
if (p >= 1 && fullText[p] === "." && UPPER_LETTER_RE.test(fullText[p - 1] ?? "")) {
|
|
1328
|
+
count++;
|
|
1329
|
+
scan = p - 1;
|
|
1330
|
+
continue;
|
|
1331
|
+
}
|
|
1332
|
+
break;
|
|
1080
1333
|
}
|
|
1081
1334
|
return count;
|
|
1082
1335
|
};
|
|
1083
1336
|
/**
|
|
1337
|
+
* True when `word` is a recognized legal-form suffix
|
|
1338
|
+
* (case-sensitive against the legal-forms vocabulary).
|
|
1339
|
+
* Used when deciding whether to cross an "and" connector
|
|
1340
|
+
* during backward extension — if the word immediately
|
|
1341
|
+
* preceding the connector is itself a legal-form suffix,
|
|
1342
|
+
* the "and" sits between two organisation names rather
|
|
1343
|
+
* than inside one ("Morgan Securities LLC and Allen &
|
|
1344
|
+
* Company LLC"), so the walk must stop there.
|
|
1345
|
+
*/
|
|
1346
|
+
const isKnownLegalFormSuffix = (word) => {
|
|
1347
|
+
if (word.length === 0) return false;
|
|
1348
|
+
return getNormalizedLegalBoundarySuffixesSync().has(word);
|
|
1349
|
+
};
|
|
1350
|
+
const isInNameLegalFormWord = (word) => {
|
|
1351
|
+
if (word.length === 0) return false;
|
|
1352
|
+
return getNormalizedInNameLegalFormWordsSync().has(word);
|
|
1353
|
+
};
|
|
1354
|
+
/**
|
|
1355
|
+
* If `pos` is immediately preceded (modulo horizontal
|
|
1356
|
+
* whitespace) by an initial-dot run like `J.P.`, `U.S.`,
|
|
1357
|
+
* or `N.A.`, return the position where the initial run
|
|
1358
|
+
* starts. Otherwise return `pos` unchanged. The run must
|
|
1359
|
+
* be word-bounded on the left so we never absorb a stray
|
|
1360
|
+
* sentence-ending dot.
|
|
1361
|
+
*/
|
|
1362
|
+
const skipInitialsBackward = (fullText, pos) => {
|
|
1363
|
+
let scan = pos - 1;
|
|
1364
|
+
while (scan >= 0) {
|
|
1365
|
+
const ch = fullText.charAt(scan);
|
|
1366
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1367
|
+
scan--;
|
|
1368
|
+
}
|
|
1369
|
+
if (scan < 0 || fullText.charAt(scan) !== ".") return pos;
|
|
1370
|
+
const scanLimit = Math.max(0, scan + 1 - 100);
|
|
1371
|
+
const head = fullText.slice(scanLimit, scan + 1);
|
|
1372
|
+
const match = /(?:\p{Lu}\.[^\S\n]?){2,}$/u.exec(head);
|
|
1373
|
+
if (match === null) return pos;
|
|
1374
|
+
const start = scanLimit + match.index;
|
|
1375
|
+
if (start > 0) {
|
|
1376
|
+
const prevCh = fullText.charAt(start - 1);
|
|
1377
|
+
if (/[\p{L}\p{M}\p{N}]/u.test(prevCh)) return pos;
|
|
1378
|
+
}
|
|
1379
|
+
return start;
|
|
1380
|
+
};
|
|
1381
|
+
/**
|
|
1084
1382
|
* Extend a match backward through uppercase words and
|
|
1085
1383
|
* lowercase connectors. Stops at start of text,
|
|
1086
1384
|
* newline, or a word that doesn't qualify.
|
|
@@ -1109,24 +1407,57 @@ const extendBackward = (fullText, matchStart, options = {}) => {
|
|
|
1109
1407
|
const isConnector = CONNECTOR_RE.test(word);
|
|
1110
1408
|
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1111
1409
|
if (isUpper) pos = wordStart;
|
|
1112
|
-
else if (isConnector) {
|
|
1113
|
-
if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
|
|
1410
|
+
else if (isConnector) if (AND_TYPE_CONNECTOR_RE.test(word)) {
|
|
1114
1411
|
const prev = findWordBefore(fullText, wordStart);
|
|
1115
1412
|
if (!prev) break;
|
|
1116
1413
|
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1414
|
+
if (isKnownLegalFormSuffix(prev.word)) break;
|
|
1415
|
+
const upperWordsBefore = countUpperWordsBefore(fullText, wordStart, suffixMode);
|
|
1416
|
+
const middleInitialBefore = hasMiddleInitialBefore(fullText, wordStart);
|
|
1417
|
+
if (upperWordsBefore <= 1 && (getClauseNounHeadsSync().has(prev.word.toLowerCase()) || getConnectorProseHeadsSync().has(prev.word.toLowerCase()))) break;
|
|
1418
|
+
if (suffixMode ? middleInitialBefore && hasSingleCapPrefixBefore(fullText, matchStart) : upperWordsBefore === 2 && !isInNameLegalFormWord(prev.word) || middleInitialBefore) break;
|
|
1117
1419
|
pos = prev.start;
|
|
1118
|
-
} else
|
|
1420
|
+
} else {
|
|
1421
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1422
|
+
if (!prev) break;
|
|
1423
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1424
|
+
pos = prev.start;
|
|
1425
|
+
}
|
|
1426
|
+
else if (isInNamePrep) {
|
|
1119
1427
|
const prev = findWordBefore(fullText, wordStart);
|
|
1120
1428
|
if (!prev) break;
|
|
1121
1429
|
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1122
1430
|
pos = prev.start;
|
|
1123
1431
|
} else break;
|
|
1124
1432
|
}
|
|
1433
|
+
pos = skipInitialsBackward(fullText, pos);
|
|
1125
1434
|
return pos;
|
|
1126
1435
|
};
|
|
1127
1436
|
const trimLeadingClause = (text) => {
|
|
1128
1437
|
let cut = -1;
|
|
1129
|
-
|
|
1438
|
+
const trims = getLeadingClauseTrimsSync();
|
|
1439
|
+
const phraseAlternation = trims.phrases.map(escapeForRegex).join("|");
|
|
1440
|
+
if (phraseAlternation.length > 0) {
|
|
1441
|
+
const phraseRe = new RegExp(`(?:^|\\s)(?:${phraseAlternation})${HSPACE}+`, "giu");
|
|
1442
|
+
for (const match of text.matchAll(phraseRe)) cut = Math.max(cut, match.index + match[0].length);
|
|
1443
|
+
}
|
|
1444
|
+
const directPrefixAlternation = trims.directPrefixes.map(escapeForRegex).join("|");
|
|
1445
|
+
if (directPrefixAlternation.length > 0) {
|
|
1446
|
+
const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
|
|
1447
|
+
for (const match of text.matchAll(directPrefixRe)) {
|
|
1448
|
+
const words = text.slice(0, match.index).match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
|
|
1449
|
+
if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
for (const match of text.matchAll(/,/gu)) {
|
|
1453
|
+
const comma = match.index;
|
|
1454
|
+
if (comma === void 0) continue;
|
|
1455
|
+
const before = text.slice(0, comma);
|
|
1456
|
+
if (!/\d/u.test(before)) continue;
|
|
1457
|
+
const after = text.slice(comma + 1);
|
|
1458
|
+
const leadingWs = after.match(/^\s*/u)?.[0].length ?? 0;
|
|
1459
|
+
if ((after.slice(leadingWs).match(/\p{Lu}[\p{L}\p{M}\p{N}]*/gu) ?? []).length >= 3) cut = Math.max(cut, comma + 1 + leadingWs);
|
|
1460
|
+
}
|
|
1130
1461
|
if (cut <= 0) return {
|
|
1131
1462
|
offset: 0,
|
|
1132
1463
|
text
|
|
@@ -1162,6 +1493,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1162
1493
|
const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
|
|
1163
1494
|
let processedStart = match.start;
|
|
1164
1495
|
let processedText = text;
|
|
1496
|
+
if (isStructuralSingleCapMatch(processedText) || fullText !== void 0 && isBareSingleCapStructuralInnerMatch(fullText, match.start, text)) continue;
|
|
1165
1497
|
let trimmed = false;
|
|
1166
1498
|
const firstWordText = firstWordMatch?.[0] ?? "";
|
|
1167
1499
|
const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
|
|
@@ -1208,56 +1540,64 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1208
1540
|
}
|
|
1209
1541
|
}
|
|
1210
1542
|
}
|
|
1211
|
-
if (processedText.includes("\n")) continue;
|
|
1543
|
+
if (processedText.includes("\n") && hasDisallowedLineBreak(processedText)) continue;
|
|
1212
1544
|
let entityStart = processedStart;
|
|
1213
1545
|
let entityText = processedText;
|
|
1214
1546
|
if (fullText && !trimmed) {
|
|
1215
|
-
const extended = extendBackward(fullText, processedStart);
|
|
1547
|
+
const extended = !BARE_SINGLE_CAP_LEGAL_FORM_RE.test(processedText) ? extendBackward(fullText, processedStart) : processedStart;
|
|
1216
1548
|
if (extended < processedStart) {
|
|
1217
1549
|
entityStart = extended;
|
|
1218
1550
|
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1219
1551
|
}
|
|
1220
1552
|
}
|
|
1221
|
-
const
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
const
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
};
|
|
1232
|
-
};
|
|
1233
|
-
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1234
|
-
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1235
|
-
if (isAllCapsMatch && fullText) {
|
|
1236
|
-
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1237
|
-
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1238
|
-
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1239
|
-
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1240
|
-
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1241
|
-
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1242
|
-
entityStart = match.start;
|
|
1243
|
-
entityText = text;
|
|
1244
|
-
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1245
|
-
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1553
|
+
for (const segment of splitEmbeddedLegalFormList(entityStart, entityText)) {
|
|
1554
|
+
entityStart = segment.entityStart;
|
|
1555
|
+
entityText = segment.entityText;
|
|
1556
|
+
const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
|
|
1557
|
+
entityStart = listTrim.entityStart;
|
|
1558
|
+
entityText = listTrim.entityText;
|
|
1559
|
+
const clauseTrim = trimLeadingClause(entityText);
|
|
1560
|
+
if (clauseTrim.offset > 0) {
|
|
1561
|
+
entityStart += clauseTrim.offset;
|
|
1562
|
+
entityText = clauseTrim.text;
|
|
1246
1563
|
}
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1564
|
+
if (entityText.includes("\n") && hasDisallowedLineBreak(entityText)) continue;
|
|
1565
|
+
const getPrefixInfo = (value) => {
|
|
1566
|
+
const prefixEnd = value.lastIndexOf(",") !== -1 ? value.lastIndexOf(",") : value.lastIndexOf(" ");
|
|
1567
|
+
return {
|
|
1568
|
+
prefixEnd,
|
|
1569
|
+
prefixPart: prefixEnd > 0 ? value.slice(0, prefixEnd).replace(/[^a-zA-ZÀ-ž]/g, "") : value.replace(/[^a-zA-ZÀ-ž]/g, "")
|
|
1570
|
+
};
|
|
1571
|
+
};
|
|
1572
|
+
let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
|
|
1573
|
+
let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1574
|
+
if (isAllCapsMatch && fullText) {
|
|
1575
|
+
const lineStart = fullText.lastIndexOf("\n", entityStart);
|
|
1576
|
+
const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
|
|
1577
|
+
const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
|
|
1578
|
+
const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
|
|
1579
|
+
if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
|
|
1580
|
+
if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
|
|
1581
|
+
entityStart = match.start;
|
|
1582
|
+
entityText = text;
|
|
1583
|
+
({prefixEnd, prefixPart} = getPrefixInfo(entityText));
|
|
1584
|
+
isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
|
|
1585
|
+
}
|
|
1586
|
+
} else if (isAllCapsMatch) continue;
|
|
1587
|
+
const lastSuffixSeparator = findLastSuffixSeparator(entityText);
|
|
1588
|
+
const rawSuffix = lastSuffixSeparator !== -1 ? entityText.slice(lastSuffixSeparator + 1) : "";
|
|
1589
|
+
const suffixClean = rawSuffix.replace(/[.,]/g, "");
|
|
1590
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1591
|
+
if (suffixClean.length <= 2 && !/\./.test(rawSuffix) && /[^\x00-\x7F]/.test(stripDocxSpaces(entityText.slice(0, lastSuffixSeparator !== -1 ? lastSuffixSeparator : entityText.length)))) continue;
|
|
1592
|
+
results.push({
|
|
1593
|
+
start: entityStart,
|
|
1594
|
+
end: entityStart + entityText.length,
|
|
1595
|
+
label: "organization",
|
|
1596
|
+
text: entityText,
|
|
1597
|
+
score: .95,
|
|
1598
|
+
source: DETECTION_SOURCES.LEGAL_FORM
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1261
1601
|
}
|
|
1262
1602
|
return results;
|
|
1263
1603
|
};
|
|
@@ -1564,20 +1904,23 @@ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, dat
|
|
|
1564
1904
|
const label = data.labels[localIdx];
|
|
1565
1905
|
if (!label) continue;
|
|
1566
1906
|
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
1907
|
+
const isExtended = extended !== null;
|
|
1567
1908
|
const end = extended?.end ?? match.end;
|
|
1568
1909
|
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
1569
1910
|
exactSpans.push({
|
|
1570
1911
|
start: match.start,
|
|
1571
1912
|
end
|
|
1572
1913
|
});
|
|
1573
|
-
|
|
1914
|
+
const entity = {
|
|
1574
1915
|
start: match.start,
|
|
1575
1916
|
end,
|
|
1576
1917
|
label,
|
|
1577
1918
|
text,
|
|
1578
1919
|
score: .9,
|
|
1579
1920
|
source: DETECTION_SOURCES.GAZETTEER
|
|
1580
|
-
}
|
|
1921
|
+
};
|
|
1922
|
+
if (isExtended) entity.sourceDetail = "gazetteer-extension";
|
|
1923
|
+
results.push(entity);
|
|
1581
1924
|
}
|
|
1582
1925
|
for (const match of allMatches) {
|
|
1583
1926
|
const idx = match.pattern;
|
|
@@ -1668,20 +2011,31 @@ const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList
|
|
|
1668
2011
|
* with per-language first names and surnames. When
|
|
1669
2012
|
* omitted, only legacy config files are used.
|
|
1670
2013
|
*/
|
|
1671
|
-
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
1672
|
-
|
|
2014
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
2015
|
+
const languageKey = languages?.toSorted().join(",") ?? "*";
|
|
2016
|
+
if (ctx.nameCorpus && ctx.nameCorpusKey === languageKey) return Promise.resolve();
|
|
2017
|
+
if (ctx.nameCorpusPromise && ctx.nameCorpusKey === languageKey) return ctx.nameCorpusPromise;
|
|
2018
|
+
ctx.nameCorpus = null;
|
|
2019
|
+
ctx.nameCorpusKey = languageKey;
|
|
1673
2020
|
const promise = (async () => {
|
|
1674
2021
|
try {
|
|
1675
|
-
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
2022
|
+
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod, commonWordsMod] = await Promise.all([
|
|
1676
2023
|
import("./names-first.mjs"),
|
|
1677
2024
|
import("./names-surnames.mjs"),
|
|
1678
2025
|
import("./names-title-tokens.mjs"),
|
|
1679
|
-
import("./names-exclusions.mjs")
|
|
2026
|
+
import("./names-exclusions.mjs"),
|
|
2027
|
+
import("./common-words-en.mjs")
|
|
1680
2028
|
]);
|
|
1681
2029
|
const firstNames = [...legacyFirstMod.default.names];
|
|
1682
|
-
if (dictionaries?.firstNames)
|
|
2030
|
+
if (dictionaries?.firstNames) {
|
|
2031
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.firstNames) : Object.entries(dictionaries.firstNames).filter(([language]) => languages.includes(language));
|
|
2032
|
+
for (const [, names] of entries) for (const name of names) firstNames.push(name);
|
|
2033
|
+
}
|
|
1683
2034
|
const surnames = [...legacySurnameMod.default.names];
|
|
1684
|
-
if (dictionaries?.surnames)
|
|
2035
|
+
if (dictionaries?.surnames) {
|
|
2036
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.surnames) : Object.entries(dictionaries.surnames).filter(([language]) => languages.includes(language));
|
|
2037
|
+
for (const [, names] of entries) for (const name of names) surnames.push(name);
|
|
2038
|
+
}
|
|
1685
2039
|
const dedup = (arr) => {
|
|
1686
2040
|
const seen = /* @__PURE__ */ new Set();
|
|
1687
2041
|
const result = [];
|
|
@@ -1692,8 +2046,9 @@ const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
|
1692
2046
|
}
|
|
1693
2047
|
return result;
|
|
1694
2048
|
};
|
|
2049
|
+
const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
|
|
1695
2050
|
const dedupFirst = dedup(firstNames);
|
|
1696
|
-
const dedupSurnames = dedup(surnames);
|
|
2051
|
+
const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
|
|
1697
2052
|
const titles = titleMod.default.tokens;
|
|
1698
2053
|
const exclusions = exclusionMod.default.words;
|
|
1699
2054
|
ctx.nameCorpus = {
|
|
@@ -2122,8 +2477,10 @@ const MIN_MONTH_NAME_LENGTH = 3;
|
|
|
2122
2477
|
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
2123
2478
|
/** Escape for use inside a regex alternation. */
|
|
2124
2479
|
const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2480
|
+
const escapeRegexPhrase = (s) => escapeRegex$2(s.trim()).replace(/\s+/g, "[^\\S\\n\\t]+");
|
|
2125
2481
|
/** Escape for use inside a regex character class. */
|
|
2126
2482
|
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
2483
|
+
const toSortedAlternation = (values) => [...new Set(values.map(escapeRegexPhrase).filter((value) => value.length > 0))].toSorted((a, b) => b.length - a.length).join("|");
|
|
2127
2484
|
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
2128
2485
|
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
2129
2486
|
const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
|
|
@@ -2134,6 +2491,7 @@ const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).ma
|
|
|
2134
2491
|
const escaped = escapeRegex$2(h);
|
|
2135
2492
|
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
2136
2493
|
}).join("|");
|
|
2494
|
+
const AMOUNT_WORDS = amount_words_default;
|
|
2137
2495
|
const toEntry = (validator, label, score) => {
|
|
2138
2496
|
const pattern = toRegex(validator).source;
|
|
2139
2497
|
if (!pattern) return null;
|
|
@@ -2171,9 +2529,14 @@ const STDNUM_ENTRIES = [
|
|
|
2171
2529
|
toEntry(at.uid, "tax identification number", .95),
|
|
2172
2530
|
toEntry(at.tin, "tax identification number", .9),
|
|
2173
2531
|
toEntry(at.businessid, "registration number", .95),
|
|
2532
|
+
toEntry(ch.uid, "registration number", .95),
|
|
2533
|
+
toEntry(au.abn, "tax identification number", .9),
|
|
2534
|
+
toEntry(au.acn, "registration number", .9),
|
|
2174
2535
|
toEntry(be.vat, "tax identification number", .95),
|
|
2175
2536
|
toEntry(be.nn, "national identification number", .9),
|
|
2176
2537
|
toEntry(nl.vat, "tax identification number", .95),
|
|
2538
|
+
toEntry(no.orgnr, "registration number", .9),
|
|
2539
|
+
toEntry(no.mva, "tax identification number", .95),
|
|
2177
2540
|
toEntry(dk.vat, "tax identification number", .95),
|
|
2178
2541
|
toEntry(dk.cpr, "national identification number", .9),
|
|
2179
2542
|
toEntry(fi.vat, "tax identification number", .95),
|
|
@@ -2203,6 +2566,7 @@ const STDNUM_ENTRIES = [
|
|
|
2203
2566
|
toEntry(cy.vat, "tax identification number", .95),
|
|
2204
2567
|
toEntry(mt.vat, "tax identification number", .95),
|
|
2205
2568
|
toEntry(lu.vat, "tax identification number", .95),
|
|
2569
|
+
toEntry(us.ein, "tax identification number", .9),
|
|
2206
2570
|
toEntry(br.cpf, "tax identification number", .95),
|
|
2207
2571
|
toEntry(br.cnpj, "tax identification number", .95)
|
|
2208
2572
|
].filter((e) => e !== null);
|
|
@@ -2315,6 +2679,30 @@ const ES_CIF = {
|
|
|
2315
2679
|
score: .95,
|
|
2316
2680
|
validator: es.cif
|
|
2317
2681
|
};
|
|
2682
|
+
const AU_ABN_FORMATTED = {
|
|
2683
|
+
pattern: `\\b\\d{2}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
|
|
2684
|
+
label: "tax identification number",
|
|
2685
|
+
score: .95,
|
|
2686
|
+
validator: au.abn
|
|
2687
|
+
};
|
|
2688
|
+
const NO_ORGNR_FORMATTED = {
|
|
2689
|
+
pattern: `\\b\\d{3}[^\\S\\n]\\d{3}[^\\S\\n]\\d{3}\\b`,
|
|
2690
|
+
label: "registration number",
|
|
2691
|
+
score: .9,
|
|
2692
|
+
validator: no.orgnr
|
|
2693
|
+
};
|
|
2694
|
+
const NO_MVA_FORMATTED = {
|
|
2695
|
+
pattern: "\\bNO[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?\\d{3}[^\\S\\n]?MVA\\b",
|
|
2696
|
+
label: "tax identification number",
|
|
2697
|
+
score: .95,
|
|
2698
|
+
validator: no.mva
|
|
2699
|
+
};
|
|
2700
|
+
const US_EIN_FORMATTED = {
|
|
2701
|
+
pattern: `\\b\\d{2}${DASH}\\d{7}\\b`,
|
|
2702
|
+
label: "tax identification number",
|
|
2703
|
+
score: .95,
|
|
2704
|
+
validator: us.ein
|
|
2705
|
+
};
|
|
2318
2706
|
const BR_CPF_FORMATTED = {
|
|
2319
2707
|
pattern: `\\b\\d{3}\\.\\d{3}\\.\\d{3}${DASH}\\d{2}\\b`,
|
|
2320
2708
|
label: "tax identification number",
|
|
@@ -2347,6 +2735,58 @@ const SHORT_TLDS = "de|at|be|se|fi|dk|no|it|uk";
|
|
|
2347
2735
|
const HOST_LABEL = `[a-zA-Z0-9](?:[a-zA-Z0-9\\-]*[a-zA-Z0-9])?`;
|
|
2348
2736
|
const BARE_HOST = `\\b[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]`;
|
|
2349
2737
|
const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
|
|
2738
|
+
const BARE_DOMAIN = {
|
|
2739
|
+
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
2740
|
+
label: "url",
|
|
2741
|
+
score: .9
|
|
2742
|
+
};
|
|
2743
|
+
const IPV6_ADDRESS = {
|
|
2744
|
+
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
2745
|
+
label: "ip address",
|
|
2746
|
+
score: 1
|
|
2747
|
+
};
|
|
2748
|
+
const MAC_ADDRESS = {
|
|
2749
|
+
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
2750
|
+
label: "mac address",
|
|
2751
|
+
score: 1
|
|
2752
|
+
};
|
|
2753
|
+
const UK_POSTCODE = {
|
|
2754
|
+
pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
|
|
2755
|
+
label: "address",
|
|
2756
|
+
score: .9
|
|
2757
|
+
};
|
|
2758
|
+
const UK_NINO = {
|
|
2759
|
+
pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
|
|
2760
|
+
label: "social security number",
|
|
2761
|
+
score: .95,
|
|
2762
|
+
validator: gb.nino
|
|
2763
|
+
};
|
|
2764
|
+
const TIME_12H = {
|
|
2765
|
+
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
2766
|
+
label: "date",
|
|
2767
|
+
score: .9
|
|
2768
|
+
};
|
|
2769
|
+
const PERCENT_TOKEN = `${`(?:[+${DASH_INNER}])?(?:\\d{1,3}(?:[.,]\\d{3})+(?:[.,]\\d{1,4})?|\\d+(?:[.,]\\d{1,4})?)`}[^\\S\\n]{0,2}%`;
|
|
2770
|
+
const PERCENT_RANGE_NUMBER = `\\d+(?:[.,]\\d{1,4})?`;
|
|
2771
|
+
const PERCENT_RANGE = `${PERCENT_RANGE_NUMBER}[^\\S\\n]*${DASH}[^\\S\\n]*${PERCENT_RANGE_NUMBER}[^\\S\\n]{0,2}%`;
|
|
2772
|
+
const buildPercentWordPattern = (config) => {
|
|
2773
|
+
const phrases = [];
|
|
2774
|
+
for (const entry of config.percentages ?? []) {
|
|
2775
|
+
const ones = entry.ones.map(escapeRegex$2);
|
|
2776
|
+
const standalone = (entry.standalone ?? []).map(escapeRegex$2);
|
|
2777
|
+
const baseWords = [
|
|
2778
|
+
...ones,
|
|
2779
|
+
...entry.teens,
|
|
2780
|
+
...entry.tens
|
|
2781
|
+
].map(escapeRegex$2);
|
|
2782
|
+
const compoundSeparator = entry.allowSpaceCompoundSeparator ? `(?:${DASH}|[^\\S\\n]+)` : DASH;
|
|
2783
|
+
const compound = `(?:${baseWords.join("|")})` + (ones.length > 0 ? `(?:${compoundSeparator}(?:${ones.join("|")}))?` : "");
|
|
2784
|
+
const word = `(?:${[...standalone, compound].join("|")})`;
|
|
2785
|
+
const keyword = `(?:${entry.keywords.map(escapeRegex$2).join("|")})`;
|
|
2786
|
+
phrases.push(`${word}[^\\S\\n]+${keyword}`);
|
|
2787
|
+
}
|
|
2788
|
+
return phrases.length > 0 ? `(?i:(?:${phrases.join("|")}))` : "(?!)";
|
|
2789
|
+
};
|
|
2350
2790
|
/**
|
|
2351
2791
|
* All static PII regex definitions. Scanned in a
|
|
2352
2792
|
* single pass by @stll/regex-set (Rust DFA).
|
|
@@ -2383,41 +2823,25 @@ const ALL_REGEX_DEFS = [
|
|
|
2383
2823
|
ES_DNI,
|
|
2384
2824
|
ES_NIE,
|
|
2385
2825
|
ES_CIF,
|
|
2826
|
+
AU_ABN_FORMATTED,
|
|
2827
|
+
NO_ORGNR_FORMATTED,
|
|
2828
|
+
NO_MVA_FORMATTED,
|
|
2829
|
+
US_EIN_FORMATTED,
|
|
2386
2830
|
BR_CPF_FORMATTED,
|
|
2387
2831
|
BR_CNPJ_FORMATTED,
|
|
2388
2832
|
BR_RG_WITH_SSP,
|
|
2389
2833
|
BR_OAB,
|
|
2390
2834
|
URL,
|
|
2835
|
+
IPV6_ADDRESS,
|
|
2836
|
+
MAC_ADDRESS,
|
|
2837
|
+
BARE_DOMAIN,
|
|
2838
|
+
UK_POSTCODE,
|
|
2839
|
+
UK_NINO,
|
|
2840
|
+
TIME_12H,
|
|
2391
2841
|
{
|
|
2392
|
-
pattern:
|
|
2393
|
-
label: "
|
|
2394
|
-
score:
|
|
2395
|
-
},
|
|
2396
|
-
{
|
|
2397
|
-
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
2398
|
-
label: "mac address",
|
|
2399
|
-
score: 1
|
|
2400
|
-
},
|
|
2401
|
-
{
|
|
2402
|
-
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
2403
|
-
label: "url",
|
|
2404
|
-
score: .9
|
|
2405
|
-
},
|
|
2406
|
-
{
|
|
2407
|
-
pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
|
|
2408
|
-
label: "address",
|
|
2409
|
-
score: .9
|
|
2410
|
-
},
|
|
2411
|
-
{
|
|
2412
|
-
pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
|
|
2413
|
-
label: "social security number",
|
|
2414
|
-
score: .95,
|
|
2415
|
-
validator: gb.nino
|
|
2416
|
-
},
|
|
2417
|
-
{
|
|
2418
|
-
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
2419
|
-
label: "date",
|
|
2420
|
-
score: .9
|
|
2842
|
+
pattern: `(?<![\\p{L}\\p{N}_.,])(?:${buildPercentWordPattern(AMOUNT_WORDS)}[^\\S\\n]*\\([^\\S\\n]*${PERCENT_TOKEN}[^\\S\\n]*\\)|${PERCENT_RANGE}|${PERCENT_TOKEN})(?![\\p{L}\\p{N}_])`,
|
|
2843
|
+
label: "monetary amount",
|
|
2844
|
+
score: .85
|
|
2421
2845
|
},
|
|
2422
2846
|
...STDNUM_ENTRIES
|
|
2423
2847
|
];
|
|
@@ -2491,6 +2915,41 @@ const DATE_PATTERN_META = Object.freeze({
|
|
|
2491
2915
|
label: "date",
|
|
2492
2916
|
score: 1
|
|
2493
2917
|
});
|
|
2918
|
+
const buildMagnitudePattern = (config) => {
|
|
2919
|
+
const words = [];
|
|
2920
|
+
const caseInsensitiveAbbreviations = [];
|
|
2921
|
+
const caseSensitiveAbbreviations = [];
|
|
2922
|
+
for (const entry of config.magnitudeSuffixes ?? []) {
|
|
2923
|
+
words.push(...entry.words ?? []);
|
|
2924
|
+
caseInsensitiveAbbreviations.push(...entry.abbreviationsCaseInsensitive ?? []);
|
|
2925
|
+
caseSensitiveAbbreviations.push(...entry.abbreviationsCaseSensitive ?? []);
|
|
2926
|
+
}
|
|
2927
|
+
const branches = [];
|
|
2928
|
+
const wordsAlt = toSortedAlternation(words);
|
|
2929
|
+
const abbreviationCiAlt = toSortedAlternation(caseInsensitiveAbbreviations);
|
|
2930
|
+
const abbreviationCsAlt = toSortedAlternation(caseSensitiveAbbreviations);
|
|
2931
|
+
if (wordsAlt) branches.push(`[^\\S\\n\\t]+(?i:(?:${wordsAlt}))\\b`);
|
|
2932
|
+
if (abbreviationCiAlt) branches.push(`[^\\S\\n\\t]?(?i:${abbreviationCiAlt})\\b`);
|
|
2933
|
+
if (abbreviationCsAlt) branches.push(`[^\\S\\n\\t]?(?:${abbreviationCsAlt})\\b`);
|
|
2934
|
+
return branches.length > 0 ? `(?:${branches.join("|")})?` : "";
|
|
2935
|
+
};
|
|
2936
|
+
const buildQuantityFollowerGuard = (config) => {
|
|
2937
|
+
const modifiers = [];
|
|
2938
|
+
const nouns = [];
|
|
2939
|
+
for (const entry of config.shareQuantityTerms ?? []) {
|
|
2940
|
+
modifiers.push(...entry.modifiers ?? []);
|
|
2941
|
+
nouns.push(...entry.nouns);
|
|
2942
|
+
}
|
|
2943
|
+
const modifierAlt = toSortedAlternation(modifiers);
|
|
2944
|
+
const nounAlt = toSortedAlternation(nouns);
|
|
2945
|
+
if (!nounAlt) return "";
|
|
2946
|
+
return `(?![^\\S\\n\\t]+(?i:${modifierAlt ? `(?:(?:${modifierAlt})[^\\S\\n\\t]+){0,3}` : ""}(?:${nounAlt}))\\b)`;
|
|
2947
|
+
};
|
|
2948
|
+
const buildFinancialLexicons = (config) => Object.freeze({
|
|
2949
|
+
magnitude: buildMagnitudePattern(config),
|
|
2950
|
+
quantityFollowerGuard: buildQuantityFollowerGuard(config)
|
|
2951
|
+
});
|
|
2952
|
+
const FINANCIAL_LEXICONS = buildFinancialLexicons(AMOUNT_WORDS);
|
|
2494
2953
|
/**
|
|
2495
2954
|
* Build symbol character class, code alternation,
|
|
2496
2955
|
* and local-name alternation from currencies.json,
|
|
@@ -2530,9 +2989,13 @@ const buildCurrencyPatterns = (data) => {
|
|
|
2530
2989
|
const patterns = [];
|
|
2531
2990
|
const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
|
|
2532
2991
|
const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
|
|
2533
|
-
|
|
2534
|
-
if (
|
|
2535
|
-
if (trailingAlt) patterns.push(`\\b
|
|
2992
|
+
const MAGNITUDE = FINANCIAL_LEXICONS.magnitude;
|
|
2993
|
+
if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${MAGNITUDE}${END}`);
|
|
2994
|
+
if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${MAGNITUDE}${END}`);
|
|
2995
|
+
if (trailingAlt) {
|
|
2996
|
+
const optionalLeadingSymbol = symbols ? `(?<![\\p{L}\\p{N}_])(?:[${symbols}][^\\S\\n\\t]?)?` : "\\b";
|
|
2997
|
+
patterns.push(`${optionalLeadingSymbol}${NUM}${DECIMAL}${MAGNITUDE}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${FINANCIAL_LEXICONS.quantityFollowerGuard}${END}`);
|
|
2998
|
+
}
|
|
2536
2999
|
return patterns;
|
|
2537
3000
|
};
|
|
2538
3001
|
/** Cached promise for currency patterns. Loaded once. */
|
|
@@ -2846,10 +3309,19 @@ const stripQuotes = (value) => {
|
|
|
2846
3309
|
text: stripped
|
|
2847
3310
|
};
|
|
2848
3311
|
};
|
|
2849
|
-
/**
|
|
3312
|
+
/**
|
|
3313
|
+
* Hard stop characters for to-next-comma scanning. A closing
|
|
3314
|
+
* parenthesis or closing bracket terminates a clause just like
|
|
3315
|
+
* an opening one: `State of New York or any other jurisdiction)`
|
|
3316
|
+
* is the tail of a parenthesised insertion, not the start of a
|
|
3317
|
+
* larger phrase that should be absorbed into a jurisdiction span.
|
|
3318
|
+
*/
|
|
2850
3319
|
const COMMA_STOP_CHARS = new Set([
|
|
2851
3320
|
"\n",
|
|
2852
3321
|
"(",
|
|
3322
|
+
")",
|
|
3323
|
+
"[",
|
|
3324
|
+
"]",
|
|
2853
3325
|
" ",
|
|
2854
3326
|
";"
|
|
2855
3327
|
]);
|
|
@@ -3015,6 +3487,27 @@ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STO
|
|
|
3015
3487
|
const warmAddressStopKeywords = async () => {
|
|
3016
3488
|
await loadAddressStopKeywords();
|
|
3017
3489
|
};
|
|
3490
|
+
const MAX_TRIGGER_VALUE_LEN = 100;
|
|
3491
|
+
const MIN_TRIGGER_PHONE_DIGITS = 5;
|
|
3492
|
+
const PHONE_VALUE_START_RE = /^[+(\d]/;
|
|
3493
|
+
const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
|
|
3494
|
+
const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
3495
|
+
const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
3496
|
+
const capAtWordBoundary = (valueText, cap) => {
|
|
3497
|
+
let capped = cap;
|
|
3498
|
+
const isWordChar = (i) => /[\p{L}\p{N}]/u.test(valueText[i] ?? "");
|
|
3499
|
+
while (capped > 0 && isWordChar(capped - 1) && isWordChar(capped)) capped--;
|
|
3500
|
+
return capped;
|
|
3501
|
+
};
|
|
3502
|
+
const isPlausiblePhoneTriggerValue = (value) => {
|
|
3503
|
+
const trimmed = value.trimStart();
|
|
3504
|
+
if (!PHONE_VALUE_START_RE.test(trimmed)) return false;
|
|
3505
|
+
if (ISO_DATE_PREFIX_RE.test(trimmed)) return false;
|
|
3506
|
+
if (INLINE_FIELD_LABEL_RE.test(trimmed)) return false;
|
|
3507
|
+
let digits = 0;
|
|
3508
|
+
for (const ch of trimmed) if (/\d/.test(ch)) digits++;
|
|
3509
|
+
return digits >= MIN_TRIGGER_PHONE_DIGITS;
|
|
3510
|
+
};
|
|
3018
3511
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
3019
3512
|
const remaining = text.slice(triggerEnd);
|
|
3020
3513
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
@@ -3026,21 +3519,11 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3026
3519
|
const stopWords = strategy.stopWords ?? [];
|
|
3027
3520
|
const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
|
|
3028
3521
|
let end = 0;
|
|
3029
|
-
let foundStop = false;
|
|
3030
3522
|
while (end < valueText.length) {
|
|
3031
3523
|
const ch = valueText[end];
|
|
3032
|
-
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch))
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
}
|
|
3036
|
-
if (ch === "." && isSentenceTerminator(valueText, end)) {
|
|
3037
|
-
foundStop = true;
|
|
3038
|
-
break;
|
|
3039
|
-
}
|
|
3040
|
-
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
|
|
3041
|
-
foundStop = true;
|
|
3042
|
-
break;
|
|
3043
|
-
}
|
|
3524
|
+
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) break;
|
|
3525
|
+
if (ch === "." && isSentenceTerminator(valueText, end)) break;
|
|
3526
|
+
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) break;
|
|
3044
3527
|
if (ch === ",") {
|
|
3045
3528
|
const afterComma = valueText.slice(end);
|
|
3046
3529
|
if (DECIMAL_COMMA_RE.test(afterComma)) {
|
|
@@ -3052,12 +3535,12 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3052
3535
|
end += degreeMatch[0].length;
|
|
3053
3536
|
continue;
|
|
3054
3537
|
}
|
|
3055
|
-
foundStop = true;
|
|
3056
3538
|
break;
|
|
3057
3539
|
}
|
|
3058
3540
|
end++;
|
|
3059
3541
|
}
|
|
3060
|
-
|
|
3542
|
+
const lengthCap = strategy.maxLength ?? 100;
|
|
3543
|
+
if (end > lengthCap) end = capAtWordBoundary(valueText, lengthCap);
|
|
3061
3544
|
const rawSlice = valueText.slice(0, end);
|
|
3062
3545
|
const extracted = rawSlice.trim();
|
|
3063
3546
|
if (extracted.length === 0) return null;
|
|
@@ -3071,12 +3554,24 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3071
3554
|
case "to-end-of-line": {
|
|
3072
3555
|
const consumed = remaining.length - valueText.length;
|
|
3073
3556
|
if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
|
|
3074
|
-
const LINE_STOPS = ["\n"];
|
|
3557
|
+
const LINE_STOPS = ["\n", " "];
|
|
3075
3558
|
let end = valueText.length;
|
|
3559
|
+
let foundLineStop = false;
|
|
3076
3560
|
for (const ch of LINE_STOPS) {
|
|
3077
3561
|
const idx = valueText.indexOf(ch);
|
|
3078
|
-
if (idx !== -1 && idx < end)
|
|
3562
|
+
if (idx !== -1 && idx < end) {
|
|
3563
|
+
end = idx;
|
|
3564
|
+
foundLineStop = true;
|
|
3565
|
+
}
|
|
3566
|
+
}
|
|
3567
|
+
if (label === "phone number") {
|
|
3568
|
+
const inlineLabel = INLINE_FIELD_LABEL_STOP_RE.exec(valueText.slice(0, end));
|
|
3569
|
+
if (inlineLabel) {
|
|
3570
|
+
end = inlineLabel.index;
|
|
3571
|
+
foundLineStop = true;
|
|
3572
|
+
}
|
|
3079
3573
|
}
|
|
3574
|
+
if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
|
|
3080
3575
|
const rawSlice = valueText.slice(0, end);
|
|
3081
3576
|
const extracted = rawSlice.trim();
|
|
3082
3577
|
if (extracted.length === 0) return null;
|
|
@@ -3256,6 +3751,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
3256
3751
|
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
3257
3752
|
if (value) {
|
|
3258
3753
|
if (!applyValidations(value.text, rule.validations)) continue;
|
|
3754
|
+
if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
|
|
3259
3755
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
3260
3756
|
const entityEnd = value.end;
|
|
3261
3757
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
@@ -3597,7 +4093,7 @@ const normalizeHomoglyphs = (text) => {
|
|
|
3597
4093
|
//#endregion
|
|
3598
4094
|
//#region src/filters/false-positives.ts
|
|
3599
4095
|
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
3600
|
-
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
4096
|
+
const POSTAL_CODE_RE$1 = /\d{3}\s?\d{2}/;
|
|
3601
4097
|
const HAS_DIGIT_RE = /\d/;
|
|
3602
4098
|
const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
|
|
3603
4099
|
const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
|
|
@@ -3662,7 +4158,7 @@ const normalizeEntity = (entity) => {
|
|
|
3662
4158
|
text
|
|
3663
4159
|
};
|
|
3664
4160
|
};
|
|
3665
|
-
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
4161
|
+
const EMPTY_GENERIC_ROLES$1 = /* @__PURE__ */ new Set();
|
|
3666
4162
|
/**
|
|
3667
4163
|
* Load generic-roles.json and cache the result on the
|
|
3668
4164
|
* given context. Must be awaited during pipeline init
|
|
@@ -3686,14 +4182,14 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3686
4182
|
return ctx.genericRolesPromise;
|
|
3687
4183
|
};
|
|
3688
4184
|
/** Sync accessor — returns empty set before init. */
|
|
3689
|
-
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
4185
|
+
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES$1;
|
|
3690
4186
|
const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
|
|
3691
4187
|
let _streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3692
4188
|
let _streetTypesPromise = null;
|
|
3693
4189
|
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3694
4190
|
const loadStreetTypeRegex = async () => {
|
|
3695
4191
|
try {
|
|
3696
|
-
const data = (await import("./address-street-types.mjs")).default ?? {};
|
|
4192
|
+
const data = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
|
|
3697
4193
|
const words = /* @__PURE__ */ new Set();
|
|
3698
4194
|
for (const [key, val] of Object.entries(data)) {
|
|
3699
4195
|
if (key.startsWith("_")) continue;
|
|
@@ -3762,7 +4258,7 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3762
4258
|
}
|
|
3763
4259
|
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
|
|
3764
4260
|
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
3765
|
-
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
4261
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE$1.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3766
4262
|
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3767
4263
|
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
|
|
3768
4264
|
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
@@ -3826,6 +4322,42 @@ const loadAllowList = (ctx) => {
|
|
|
3826
4322
|
const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
3827
4323
|
/** Sync accessor — returns empty set before init. */
|
|
3828
4324
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
4325
|
+
let commonWordsPromise = null;
|
|
4326
|
+
let commonWordsCache = null;
|
|
4327
|
+
const loadCommonWords = () => {
|
|
4328
|
+
if (commonWordsCache) return Promise.resolve(commonWordsCache);
|
|
4329
|
+
if (commonWordsPromise) return commonWordsPromise;
|
|
4330
|
+
commonWordsPromise = (async () => {
|
|
4331
|
+
try {
|
|
4332
|
+
const mod = await import("./common-words-en.mjs");
|
|
4333
|
+
const set = new Set((mod.default?.words ?? []).map((word) => word.toLowerCase()));
|
|
4334
|
+
commonWordsCache = set;
|
|
4335
|
+
return set;
|
|
4336
|
+
} catch {
|
|
4337
|
+
const empty = /* @__PURE__ */ new Set();
|
|
4338
|
+
commonWordsCache = empty;
|
|
4339
|
+
return empty;
|
|
4340
|
+
}
|
|
4341
|
+
})();
|
|
4342
|
+
return commonWordsPromise;
|
|
4343
|
+
};
|
|
4344
|
+
/**
|
|
4345
|
+
* Curated dictionary entries that are pure dotted
|
|
4346
|
+
* single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
|
|
4347
|
+
* need targeted suffix guards. The AC search matches
|
|
4348
|
+
* case-insensitively on token boundaries where `.` is not
|
|
4349
|
+
* a word character, so `S.C.` can match inside `U.S.C.`.
|
|
4350
|
+
* Two-segment non-address aliases are too noisy and are
|
|
4351
|
+
* dropped at build time; longer official aliases stay
|
|
4352
|
+
* searchable and are only suppressed when the source text
|
|
4353
|
+
* shows they are the tail of a longer dotted token.
|
|
4354
|
+
* Caller-supplied custom entries are exempted.
|
|
4355
|
+
*/
|
|
4356
|
+
const DOTTED_ACRONYM_RE = /^(?=.{3,}$)\p{L}(?:\.\p{L}){0,3}\.?$/u;
|
|
4357
|
+
const isCuratedNoiseAcronym = (normalized) => DOTTED_ACRONYM_RE.test(normalized);
|
|
4358
|
+
const dottedAcronymSegmentCount = (normalized) => normalized.split(".").filter(Boolean).length;
|
|
4359
|
+
const isShortCuratedNoiseAcronym = (normalized) => isCuratedNoiseAcronym(normalized) && dottedAcronymSegmentCount(normalized) <= 2;
|
|
4360
|
+
const isDottedAcronymSuffixCollision = (fullText, start, matchText) => isCuratedNoiseAcronym(matchText) && /[\p{L}]\.$/u.test(fullText.slice(Math.max(0, start - 2), start));
|
|
3829
4361
|
/**
|
|
3830
4362
|
* Common EU given names present in the stopwords-iso dataset
|
|
3831
4363
|
* but absent from the first-name corpus. Without this
|
|
@@ -3962,7 +4494,7 @@ let streetTypeReLoaded = false;
|
|
|
3962
4494
|
const loadStreetTypeRe = async () => {
|
|
3963
4495
|
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
3964
4496
|
try {
|
|
3965
|
-
const config = (await import("./address-street-types.mjs")).default ?? {};
|
|
4497
|
+
const config = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
|
|
3966
4498
|
const words = [];
|
|
3967
4499
|
for (const value of Object.values(config)) {
|
|
3968
4500
|
if (!Array.isArray(value)) continue;
|
|
@@ -4040,6 +4572,21 @@ const SENTENCE_STARTER_WORDS = new Set([
|
|
|
4040
4572
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
4041
4573
|
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
4042
4574
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
4575
|
+
const getCityEntries = (dictionaries, allowedCountries) => {
|
|
4576
|
+
const byCountry = dictionaries?.citiesByCountry;
|
|
4577
|
+
if (!byCountry) return dictionaries?.cities ?? [];
|
|
4578
|
+
const result = [];
|
|
4579
|
+
const append = (entries) => {
|
|
4580
|
+
if (!entries) return;
|
|
4581
|
+
for (const entry of entries) result.push(entry);
|
|
4582
|
+
};
|
|
4583
|
+
if (allowedCountries === null) {
|
|
4584
|
+
for (const entries of Object.values(byCountry)) append(entries);
|
|
4585
|
+
return result;
|
|
4586
|
+
}
|
|
4587
|
+
for (const country of allowedCountries) append(byCountry[country.toUpperCase()]);
|
|
4588
|
+
return result;
|
|
4589
|
+
};
|
|
4043
4590
|
/**
|
|
4044
4591
|
* Resolve which dictionaries to load based on country
|
|
4045
4592
|
* and category filters, then build the deny list data.
|
|
@@ -4051,21 +4598,24 @@ const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^
|
|
|
4051
4598
|
* Returns null if no dictionaries are provided.
|
|
4052
4599
|
*/
|
|
4053
4600
|
const buildDenyList = async (config, ctx = defaultContext) => {
|
|
4054
|
-
await initNameCorpus(ctx, config.dictionaries);
|
|
4601
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
4055
4602
|
await Promise.all([
|
|
4056
4603
|
loadStopwords(ctx),
|
|
4057
4604
|
loadAllowList(ctx),
|
|
4058
4605
|
loadPersonStopwords(ctx),
|
|
4059
4606
|
loadAddressStopwords(ctx),
|
|
4607
|
+
loadCommonWords(),
|
|
4060
4608
|
loadStreetTypeRe(),
|
|
4061
4609
|
loadGenericRoles(ctx)
|
|
4062
4610
|
]);
|
|
4611
|
+
const commonWords = await loadCommonWords();
|
|
4063
4612
|
const dictionaries = config.dictionaries;
|
|
4064
4613
|
const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
|
|
4065
|
-
const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
|
|
4066
4614
|
const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
|
|
4067
|
-
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
4068
4615
|
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
4616
|
+
const cityEntries = getCityEntries(dictionaries, allowedCountries);
|
|
4617
|
+
const hasCities = cityEntries.length > 0;
|
|
4618
|
+
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
4069
4619
|
const excluded = config.denyListExcludeCategories;
|
|
4070
4620
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
4071
4621
|
const patternList = [];
|
|
@@ -4077,6 +4627,10 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4077
4627
|
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
4078
4628
|
if (normalized.length === 0) return;
|
|
4079
4629
|
const lower = normalized.toLowerCase();
|
|
4630
|
+
if (source !== "custom-deny-list" && label !== "address") {
|
|
4631
|
+
if (SINGLE_WORD_RE.test(normalized) && commonWords.has(lower)) return;
|
|
4632
|
+
if (isShortCuratedNoiseAcronym(normalized)) return;
|
|
4633
|
+
}
|
|
4080
4634
|
const existing = patternIndex.get(lower);
|
|
4081
4635
|
if (existing !== void 0) {
|
|
4082
4636
|
if (!labelList[existing].includes(label)) labelList[existing].push(label);
|
|
@@ -4093,10 +4647,12 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4093
4647
|
if (hasDenyList) {
|
|
4094
4648
|
const denyListData = dictionaries.denyList;
|
|
4095
4649
|
const metaData = dictionaries.denyListMeta;
|
|
4650
|
+
const useScopedNameCorpus = config.nameCorpusLanguages !== void 0;
|
|
4096
4651
|
for (const [id, entries] of Object.entries(denyListData)) {
|
|
4097
4652
|
const meta = metaData[id];
|
|
4098
4653
|
if (!meta) continue;
|
|
4099
4654
|
if (!config.enableNameCorpus && meta.category === "Names") continue;
|
|
4655
|
+
if (useScopedNameCorpus && meta.category === "Names") continue;
|
|
4100
4656
|
if (excludeCategories.has(meta.category)) continue;
|
|
4101
4657
|
if (allowedCountries !== null && meta.country !== null) {
|
|
4102
4658
|
if (!allowedCountries.has(meta.country)) continue;
|
|
@@ -4104,7 +4660,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4104
4660
|
for (const entry of entries) addDenyListEntry(entry, meta.label);
|
|
4105
4661
|
}
|
|
4106
4662
|
}
|
|
4107
|
-
if (hasCities && !excludeCategories.has("Places")) for (const entry of
|
|
4663
|
+
if (hasCities && !excludeCategories.has("Places")) for (const entry of cityEntries) addDenyListEntry(entry, "address", "city");
|
|
4108
4664
|
if (hasCustomDenyList) for (const entry of config.customDenyList) {
|
|
4109
4665
|
addDenyListEntry(entry.value, entry.label, "custom-deny-list");
|
|
4110
4666
|
for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
|
|
@@ -4153,6 +4709,7 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
4153
4709
|
const addNameEntry = (name, source) => {
|
|
4154
4710
|
const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
|
|
4155
4711
|
if (normalized.length === 0) return;
|
|
4712
|
+
if (isCuratedNoiseAcronym(normalized)) return;
|
|
4156
4713
|
const lower = normalized.toLowerCase();
|
|
4157
4714
|
const existing = patternIndex.get(lower);
|
|
4158
4715
|
if (existing !== void 0) {
|
|
@@ -4198,8 +4755,8 @@ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
|
|
|
4198
4755
|
* the search instance was built on a different context
|
|
4199
4756
|
* (e.g. cachedSearch).
|
|
4200
4757
|
*/
|
|
4201
|
-
const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
4202
|
-
await initNameCorpus(ctx, dictionaries);
|
|
4758
|
+
const ensureDenyListData = async (ctx = defaultContext, dictionaries, nameCorpusLanguages) => {
|
|
4759
|
+
await initNameCorpus(ctx, dictionaries, nameCorpusLanguages);
|
|
4203
4760
|
await Promise.all([
|
|
4204
4761
|
loadStopwords(ctx),
|
|
4205
4762
|
loadAllowList(ctx),
|
|
@@ -4239,12 +4796,14 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
4239
4796
|
const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
|
|
4240
4797
|
const customLabels = customEdgesAreValid ? customPatternLabels : [];
|
|
4241
4798
|
if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
|
|
4242
|
-
const
|
|
4243
|
-
|
|
4799
|
+
const acronymMatchesAcronym = !(pattern.length > 0 && pattern.length <= 5 && ALL_UPPER_RE.test(pattern)) || ALL_UPPER_RE.test(matchText);
|
|
4800
|
+
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && acronymMatchesAcronym && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
|
|
4801
|
+
const filteredCuratedLabels = isDottedAcronymSuffixCollision(fullText, match.start, matchText) ? [] : curatedLabels;
|
|
4802
|
+
if (filteredCuratedLabels.length === 0 && customLabels.length === 0) continue;
|
|
4244
4803
|
const entry = {
|
|
4245
4804
|
start: match.start,
|
|
4246
4805
|
end: match.end,
|
|
4247
|
-
labels:
|
|
4806
|
+
labels: filteredCuratedLabels,
|
|
4248
4807
|
customLabels,
|
|
4249
4808
|
sources,
|
|
4250
4809
|
text: matchText,
|
|
@@ -4310,6 +4869,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
4310
4869
|
const first = chain.at(0);
|
|
4311
4870
|
const last = chain.at(-1);
|
|
4312
4871
|
if (!first || !last) continue;
|
|
4872
|
+
if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
|
|
4313
4873
|
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
4314
4874
|
const score = chain.length >= 2 ? .9 : .5;
|
|
4315
4875
|
if (chain.length === 1) {
|
|
@@ -4375,7 +4935,7 @@ const extendCityDistricts = (entities, fullText) => {
|
|
|
4375
4935
|
entity.text = fullText.slice(entity.start, entity.end);
|
|
4376
4936
|
}
|
|
4377
4937
|
const afterDistrict = fullText.slice(entity.end);
|
|
4378
|
-
const dashDistrictM = /^[\
|
|
4938
|
+
const dashDistrictM = /^[ \t]{1,4}[-–][ \t]*(\p{Lu}\p{Ll}+)/u.exec(afterDistrict);
|
|
4379
4939
|
if (dashDistrictM && !dashDistrictM[0].includes("\n")) {
|
|
4380
4940
|
entity.end += dashDistrictM[0].length;
|
|
4381
4941
|
entity.text = fullText.slice(entity.start, entity.end);
|
|
@@ -4405,6 +4965,95 @@ const extendCityDistricts = (entities, fullText) => {
|
|
|
4405
4965
|
* by a capitalized word (for "Miroslav Braňka" when
|
|
4406
4966
|
* only "Braňka" matched).
|
|
4407
4967
|
*/
|
|
4968
|
+
/**
|
|
4969
|
+
* Defined-term marker: an opening typographic or straight
|
|
4970
|
+
* quote enclosing the chain start, AND a closing quote
|
|
4971
|
+
* within a short window followed by a
|
|
4972
|
+
* definitional cue (`means`, `shall mean`, `shall have
|
|
4973
|
+
* the meaning(s)`, `refers to`). Legal documents reserve
|
|
4974
|
+
* this construction for defined terms; the contents are
|
|
4975
|
+
* not personal names even when individual tokens collide
|
|
4976
|
+
* with the name corpus.
|
|
4977
|
+
*
|
|
4978
|
+
* Plain quotations like `"John Unknown" said ...` do NOT
|
|
4979
|
+
* count: there is no definitional cue, so the trailing
|
|
4980
|
+
* surname extension is still allowed to absorb `Unknown`.
|
|
4981
|
+
*/
|
|
4982
|
+
const OPENING_QUOTES = new Set([
|
|
4983
|
+
"\"",
|
|
4984
|
+
"'",
|
|
4985
|
+
"“",
|
|
4986
|
+
"„",
|
|
4987
|
+
"‟",
|
|
4988
|
+
"‘",
|
|
4989
|
+
"‛",
|
|
4990
|
+
"«"
|
|
4991
|
+
]);
|
|
4992
|
+
const CLOSING_QUOTES = new Set([
|
|
4993
|
+
"\"",
|
|
4994
|
+
"'",
|
|
4995
|
+
"”",
|
|
4996
|
+
"’",
|
|
4997
|
+
"»",
|
|
4998
|
+
"“"
|
|
4999
|
+
]);
|
|
5000
|
+
const DEFINED_TERM_CUE_RE = /^[\s,]*(?:means?|shall\s+means?|shall\s+have\s+the\s+meanings?|refers?\s+to|has\s+the\s+meanings?|is\s+defined)\b/iu;
|
|
5001
|
+
const DEFINED_TERM_LOOKAHEAD = 120;
|
|
5002
|
+
const DEFINED_TERM_LOOKBEHIND = 80;
|
|
5003
|
+
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
5004
|
+
const isLetter = (ch) => ch !== void 0 && /^\p{L}$/u.test(ch);
|
|
5005
|
+
const isApostropheInsideWord = (text, index) => isLetter(text[index - 1]) && isLetter(text[index + 1]);
|
|
5006
|
+
const isQuoteBoundary = (text, index) => {
|
|
5007
|
+
const ch = text[index];
|
|
5008
|
+
if (ch !== "'" && ch !== "’") return true;
|
|
5009
|
+
return !isApostropheInsideWord(text, index);
|
|
5010
|
+
};
|
|
5011
|
+
const findDefinedTermQuoteContent = (text, start) => {
|
|
5012
|
+
const min = Math.max(0, start - DEFINED_TERM_LOOKBEHIND);
|
|
5013
|
+
let quoteStart = -1;
|
|
5014
|
+
for (let i = start - 1; i >= min; i--) {
|
|
5015
|
+
const ch = text[i];
|
|
5016
|
+
if (ch === "\n") break;
|
|
5017
|
+
if (ch && OPENING_QUOTES.has(ch) && isQuoteBoundary(text, i)) {
|
|
5018
|
+
quoteStart = i;
|
|
5019
|
+
break;
|
|
5020
|
+
}
|
|
5021
|
+
if (ch && CLOSING_QUOTES.has(ch) && isQuoteBoundary(text, i)) break;
|
|
5022
|
+
}
|
|
5023
|
+
if (quoteStart === -1) return null;
|
|
5024
|
+
const max = Math.min(text.length, quoteStart + 1 + DEFINED_TERM_LOOKAHEAD);
|
|
5025
|
+
for (let i = start; i < max; i++) {
|
|
5026
|
+
const ch = text[i];
|
|
5027
|
+
if (!ch || !CLOSING_QUOTES.has(ch) || !isQuoteBoundary(text, i)) continue;
|
|
5028
|
+
const after = text.slice(i + 1, max);
|
|
5029
|
+
if (!DEFINED_TERM_CUE_RE.test(after)) return null;
|
|
5030
|
+
return {
|
|
5031
|
+
content: text.slice(quoteStart + 1, i),
|
|
5032
|
+
afterClosingQuote: after
|
|
5033
|
+
};
|
|
5034
|
+
}
|
|
5035
|
+
return null;
|
|
5036
|
+
};
|
|
5037
|
+
const FIRST_WORD_RE = /^\p{L}+/u;
|
|
5038
|
+
const WORD_RE = /\p{L}+/gu;
|
|
5039
|
+
const startsWithKnownFirstName = (quoteContent, ctx) => {
|
|
5040
|
+
const firstWord = FIRST_WORD_RE.exec(quoteContent.trim())?.[0];
|
|
5041
|
+
if (!firstWord) return false;
|
|
5042
|
+
return new Set(getNameCorpusFirstNames(ctx).map((name) => name.toLowerCase())).has(firstWord.toLowerCase());
|
|
5043
|
+
};
|
|
5044
|
+
const hasPersonRoleDefinition = (afterClosingQuote, ctx) => {
|
|
5045
|
+
const roleWords = afterClosingQuote.replace(DEFINED_TERM_CUE_RE, "").match(WORD_RE)?.slice(0, 8) ?? [];
|
|
5046
|
+
if (roleWords.length === 0) return false;
|
|
5047
|
+
const genericRoles = ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
5048
|
+
return roleWords.some((word) => genericRoles.has(word.toLowerCase()));
|
|
5049
|
+
};
|
|
5050
|
+
const isSuppressibleDefinedTermQuote = (text, start, ctx) => {
|
|
5051
|
+
const definedTermQuote = findDefinedTermQuoteContent(text, start);
|
|
5052
|
+
if (definedTermQuote === null) return false;
|
|
5053
|
+
const words = definedTermQuote.content.match(WORD_RE) ?? [];
|
|
5054
|
+
if (words.length >= 2 && startsWithKnownFirstName(definedTermQuote.content, ctx) && hasPersonRoleDefinition(definedTermQuote.afterClosingQuote, ctx)) return false;
|
|
5055
|
+
return words.length >= 2;
|
|
5056
|
+
};
|
|
4408
5057
|
const extendPersonName = (text, start, end, ctx) => {
|
|
4409
5058
|
let newEnd = end;
|
|
4410
5059
|
let pos = newEnd;
|
|
@@ -4415,7 +5064,7 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
4415
5064
|
if (!UPPER_START_RE.test(char)) break;
|
|
4416
5065
|
let wordEnd = wordStart;
|
|
4417
5066
|
while (wordEnd < text.length && !/\s/.test(text[wordEnd] ?? "")) wordEnd++;
|
|
4418
|
-
const stripped = text.slice(wordStart, wordEnd).replace(/[
|
|
5067
|
+
const stripped = text.slice(wordStart, wordEnd).replace(/[,;.”"’'“»]+$/, "");
|
|
4419
5068
|
if (stripped.length < 2) break;
|
|
4420
5069
|
const lower = stripped.toLowerCase();
|
|
4421
5070
|
if (getStopwords(ctx).has(lower) || getPersonStopwords(ctx).has(lower)) break;
|
|
@@ -4438,6 +5087,15 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
4438
5087
|
* cluster occasionally absorbs.
|
|
4439
5088
|
*/
|
|
4440
5089
|
const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
|
|
5090
|
+
const POSTAL_ADJACENT = `\\p{L}\\p{N}_${DASH_INNER}`;
|
|
5091
|
+
const POSTAL_CODE_RE = new RegExp(`(?<![${POSTAL_ADJACENT}])(?:\\d{3}\\s\\d{2}|\\d{2}${DASH}\\d{3}|\\d{5}${DASH}\\d{3}|\\d{5}${DASH}\\d{4})(?![${POSTAL_ADJACENT}])`, "gu");
|
|
5092
|
+
const BR_CEP_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{3}$`, "u");
|
|
5093
|
+
const US_ZIP_PLUS_FOUR_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{4}$`, "u");
|
|
5094
|
+
const US_STATE_ABBREV_BEFORE_ZIP_RE = new RegExp(`(?:^|[^A-Za-z0-9])(A[KLRZ]|C[AOT]|D[CE]|F[LM]|G[AU]|HI|I[ADLN]|K[SY]|LA|M[ADEHINOPST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\\s*,?\\s*$`, "u");
|
|
5095
|
+
const US_ZIP_CONTEXT_WINDOW = 120;
|
|
5096
|
+
const US_CITY_ZIP_GAP_RE = /^[\s,]+$/u;
|
|
5097
|
+
const HOUSE_NUMBER_BEFORE_STREET_RE = /\b\d{1,6}(?:[-/]\d{1,6})?\s+(?:\p{Lu}\p{L}+[^\S\n\t]+){0,4}$/u;
|
|
5098
|
+
const HOUSE_NUMBER_AFTER_STREET_RE = /^[^\S\n\t]+\d{1,6}(?:[-/]\d{1,6})?\b/u;
|
|
4441
5099
|
let cachedBoundaryRe = null;
|
|
4442
5100
|
const loadBoundaryWords = async () => {
|
|
4443
5101
|
try {
|
|
@@ -4452,7 +5110,7 @@ let cachedBrCepContextPromise = null;
|
|
|
4452
5110
|
const loadBrCueWords = async () => {
|
|
4453
5111
|
const sources = await Promise.all([(async () => {
|
|
4454
5112
|
try {
|
|
4455
|
-
return (await import("./address-street-types.mjs")).default["pt-br"];
|
|
5113
|
+
return (await import("./address-street-types.mjs").then((n) => n.n)).default["pt-br"];
|
|
4456
5114
|
} catch {
|
|
4457
5115
|
return;
|
|
4458
5116
|
}
|
|
@@ -4496,6 +5154,49 @@ const hasBrCueNearby = (fullText, start, end, re) => {
|
|
|
4496
5154
|
const window = fullText.slice(windowStart, windowEnd);
|
|
4497
5155
|
return new RegExp(re.source, re.flags.replace("g", "")).test(window);
|
|
4498
5156
|
};
|
|
5157
|
+
const getUsStateSeedBeforeZip = (fullText, start) => {
|
|
5158
|
+
const stateWindowStart = Math.max(0, start - 24);
|
|
5159
|
+
const stateWindow = fullText.slice(stateWindowStart, start);
|
|
5160
|
+
const match = US_STATE_ABBREV_BEFORE_ZIP_RE.exec(stateWindow);
|
|
5161
|
+
const state = match?.[1];
|
|
5162
|
+
if (!match || !state) return null;
|
|
5163
|
+
const stateOffset = match[0].indexOf(state);
|
|
5164
|
+
const stateStart = stateWindowStart + match.index + stateOffset;
|
|
5165
|
+
return {
|
|
5166
|
+
type: "state",
|
|
5167
|
+
start: stateStart,
|
|
5168
|
+
end: stateStart + state.length,
|
|
5169
|
+
text: state
|
|
5170
|
+
};
|
|
5171
|
+
};
|
|
5172
|
+
const hasHouseNumberNearStreetWord = (fullText, seed) => {
|
|
5173
|
+
if (/\d/.test(seed.text)) return true;
|
|
5174
|
+
const before = fullText.slice(Math.max(0, seed.start - 50), seed.start);
|
|
5175
|
+
if (HOUSE_NUMBER_BEFORE_STREET_RE.test(before)) return true;
|
|
5176
|
+
const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
|
|
5177
|
+
return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
|
|
5178
|
+
};
|
|
5179
|
+
const isLowercaseStreetWordInProse = (fullText, seed) => /^\p{Ll}/u.test(seed.text) && /^\s+\p{Ll}/u.test(fullText.slice(seed.end, seed.end + 16)) && !hasHouseNumberNearStreetWord(fullText, seed);
|
|
5180
|
+
const getUsZipPlusFourContext = (fullText, start, seeds) => {
|
|
5181
|
+
const stateSeed = getUsStateSeedBeforeZip(fullText, start);
|
|
5182
|
+
if (stateSeed !== null) return {
|
|
5183
|
+
stateSeed,
|
|
5184
|
+
hasContext: true
|
|
5185
|
+
};
|
|
5186
|
+
return {
|
|
5187
|
+
stateSeed: null,
|
|
5188
|
+
hasContext: seeds.some((seed) => {
|
|
5189
|
+
if (Math.abs(seed.start - start) > US_ZIP_CONTEXT_WINDOW) return false;
|
|
5190
|
+
if (seed.type === "address-trigger") return true;
|
|
5191
|
+
if (seed.type === "city" && seed.end <= start) {
|
|
5192
|
+
const gap = fullText.slice(seed.end, start);
|
|
5193
|
+
return US_CITY_ZIP_GAP_RE.test(gap);
|
|
5194
|
+
}
|
|
5195
|
+
if (seed.type === "street-word") return hasHouseNumberNearStreetWord(fullText, seed);
|
|
5196
|
+
return false;
|
|
5197
|
+
})
|
|
5198
|
+
};
|
|
5199
|
+
};
|
|
4499
5200
|
/**
|
|
4500
5201
|
* Build regex for boundary words. Matches any
|
|
4501
5202
|
* boundary word preceded by a word boundary.
|
|
@@ -4520,7 +5221,7 @@ const getBoundaryRe = async () => {
|
|
|
4520
5221
|
const buildStreetTypePatterns = async () => {
|
|
4521
5222
|
let config = {};
|
|
4522
5223
|
try {
|
|
4523
|
-
config = (await import("./address-street-types.mjs")).default;
|
|
5224
|
+
config = (await import("./address-street-types.mjs").then((n) => n.n)).default;
|
|
4524
5225
|
} catch {
|
|
4525
5226
|
return [];
|
|
4526
5227
|
}
|
|
@@ -4536,12 +5237,14 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
4536
5237
|
for (const match of allMatches) {
|
|
4537
5238
|
const idx = match.pattern;
|
|
4538
5239
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
4539
|
-
|
|
5240
|
+
const seed = {
|
|
4540
5241
|
type: "street-word",
|
|
4541
5242
|
start: match.start,
|
|
4542
5243
|
end: match.end,
|
|
4543
5244
|
text: match.text
|
|
4544
|
-
}
|
|
5245
|
+
};
|
|
5246
|
+
if (isLowercaseStreetWordInProse(fullText, seed)) continue;
|
|
5247
|
+
seeds.push(seed);
|
|
4545
5248
|
}
|
|
4546
5249
|
for (const e of existingEntities) {
|
|
4547
5250
|
if (e.label !== "address") continue;
|
|
@@ -4565,13 +5268,22 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
4565
5268
|
text: e.text
|
|
4566
5269
|
});
|
|
4567
5270
|
}
|
|
4568
|
-
const postalRe =
|
|
5271
|
+
const postalRe = POSTAL_CODE_RE;
|
|
5272
|
+
postalRe.lastIndex = 0;
|
|
4569
5273
|
let postalMatch;
|
|
4570
5274
|
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
4571
5275
|
const start = postalMatch.index;
|
|
4572
5276
|
const end = start + postalMatch[0].length;
|
|
4573
5277
|
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4574
|
-
if (
|
|
5278
|
+
if (BR_CEP_SHAPE_RE.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
|
|
5279
|
+
if (US_ZIP_PLUS_FOUR_SHAPE_RE.test(postalMatch[0])) {
|
|
5280
|
+
const usContext = getUsZipPlusFourContext(fullText, start, seeds);
|
|
5281
|
+
if (!usContext.hasContext) continue;
|
|
5282
|
+
const stateSeed = usContext.stateSeed;
|
|
5283
|
+
if (stateSeed !== null) {
|
|
5284
|
+
if (!seeds.some((seed) => seed.start === stateSeed.start && seed.end === stateSeed.end)) seeds.push(stateSeed);
|
|
5285
|
+
}
|
|
5286
|
+
}
|
|
4575
5287
|
seeds.push({
|
|
4576
5288
|
type: "postal-code",
|
|
4577
5289
|
start,
|
|
@@ -4650,6 +5362,7 @@ const scoreCluster = (cluster) => {
|
|
|
4650
5362
|
let score = .5;
|
|
4651
5363
|
if (types.has("postal-code")) score += .15;
|
|
4652
5364
|
if (types.has("city")) score += .15;
|
|
5365
|
+
if (types.has("state")) score += .15;
|
|
4653
5366
|
if (types.has("street-word")) score += .15;
|
|
4654
5367
|
if (types.has("address-trigger")) score += .1;
|
|
4655
5368
|
return Math.min(score, .95);
|
|
@@ -4670,6 +5383,7 @@ const NON_ADDRESS_LABELS = new Set([
|
|
|
4670
5383
|
]);
|
|
4671
5384
|
const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
4672
5385
|
const { start, end } = cluster;
|
|
5386
|
+
const seedTypes = new Set(cluster.seeds.map((seed) => seed.type));
|
|
4673
5387
|
let leftBound = 0;
|
|
4674
5388
|
for (const e of existingEntities) if (NON_ADDRESS_LABELS.has(e.label) && e.end <= start && e.end > leftBound) leftBound = e.end;
|
|
4675
5389
|
let leftPos = start;
|
|
@@ -4684,6 +5398,10 @@ const expandCluster = async (fullText, cluster, existingEntities) => {
|
|
|
4684
5398
|
if (fullText.slice(p + 1, leftPos).includes("\n")) break;
|
|
4685
5399
|
leftPos = p + 1;
|
|
4686
5400
|
}
|
|
5401
|
+
if (!(seedTypes.has("street-word") || seedTypes.has("house-number") || seedTypes.has("postal-code") || seedTypes.has("address-trigger"))) return {
|
|
5402
|
+
start: Math.min(leftPos, start),
|
|
5403
|
+
end
|
|
5404
|
+
};
|
|
4687
5405
|
let rightPos = end;
|
|
4688
5406
|
const remaining = fullText.slice(rightPos);
|
|
4689
5407
|
let nearestBoundary = Math.min(remaining.length, 200);
|
|
@@ -4836,7 +5554,44 @@ const BARE_STOPWORDS = new Set([
|
|
|
4836
5554
|
"Splatnost",
|
|
4837
5555
|
"Variabilní",
|
|
4838
5556
|
"Konstantní",
|
|
4839
|
-
"Specifický"
|
|
5557
|
+
"Specifický",
|
|
5558
|
+
"Section",
|
|
5559
|
+
"Sections",
|
|
5560
|
+
"Article",
|
|
5561
|
+
"Articles",
|
|
5562
|
+
"Schedule",
|
|
5563
|
+
"Schedules",
|
|
5564
|
+
"Exhibit",
|
|
5565
|
+
"Exhibits",
|
|
5566
|
+
"Annex",
|
|
5567
|
+
"Annexes",
|
|
5568
|
+
"Appendix",
|
|
5569
|
+
"Appendices",
|
|
5570
|
+
"Clause",
|
|
5571
|
+
"Clauses",
|
|
5572
|
+
"Chapter",
|
|
5573
|
+
"Chapters",
|
|
5574
|
+
"Paragraph",
|
|
5575
|
+
"Paragraphs",
|
|
5576
|
+
"Subsection",
|
|
5577
|
+
"Subsections",
|
|
5578
|
+
"Form",
|
|
5579
|
+
"Page",
|
|
5580
|
+
"Pages",
|
|
5581
|
+
"Item",
|
|
5582
|
+
"Items",
|
|
5583
|
+
"Note",
|
|
5584
|
+
"Notes",
|
|
5585
|
+
"Rule",
|
|
5586
|
+
"Rules",
|
|
5587
|
+
"Attachment",
|
|
5588
|
+
"Attachments",
|
|
5589
|
+
"Volume",
|
|
5590
|
+
"Volumes",
|
|
5591
|
+
"Book",
|
|
5592
|
+
"Books",
|
|
5593
|
+
"Part",
|
|
5594
|
+
"Parts"
|
|
4840
5595
|
]);
|
|
4841
5596
|
const NEAR_MISS_BAND = .15;
|
|
4842
5597
|
const BOOST_PER_NEIGHBOUR = .05;
|
|
@@ -4909,21 +5664,23 @@ const initPrepositions = () => {
|
|
|
4909
5664
|
};
|
|
4910
5665
|
const getAddressPreps = () => _addressPreps ?? /* @__PURE__ */ new Set();
|
|
4911
5666
|
const getTemporalPreps = () => _temporalPreps ?? /* @__PURE__ */ new Set();
|
|
4912
|
-
|
|
5667
|
+
const buildStreetAbbrevs = (data) => {
|
|
5668
|
+
const abbrevs = /* @__PURE__ */ new Set();
|
|
5669
|
+
for (const [key, words] of Object.entries(data)) {
|
|
5670
|
+
if (key.startsWith("_")) continue;
|
|
5671
|
+
if (!Array.isArray(words)) continue;
|
|
5672
|
+
for (const word of words) if (typeof word === "string" && word.includes(".")) abbrevs.add(word.toLowerCase());
|
|
5673
|
+
}
|
|
5674
|
+
return abbrevs;
|
|
5675
|
+
};
|
|
5676
|
+
let _streetAbbrevs = buildStreetAbbrevs(address_street_types_default);
|
|
4913
5677
|
let _streetAbbrevsPromise = null;
|
|
4914
5678
|
const loadStreetAbbrevs = async () => {
|
|
4915
5679
|
try {
|
|
4916
|
-
const mod = await import("./address-street-types.mjs");
|
|
4917
|
-
|
|
4918
|
-
const abbrevs = /* @__PURE__ */ new Set();
|
|
4919
|
-
for (const [key, words] of Object.entries(data)) {
|
|
4920
|
-
if (key.startsWith("_")) continue;
|
|
4921
|
-
if (!Array.isArray(words)) continue;
|
|
4922
|
-
for (const w of words) if (w.includes(".")) abbrevs.add(w.toLowerCase());
|
|
4923
|
-
}
|
|
4924
|
-
_streetAbbrevs = abbrevs;
|
|
5680
|
+
const mod = await import("./address-street-types.mjs").then((n) => n.n);
|
|
5681
|
+
_streetAbbrevs = buildStreetAbbrevs(mod.default ?? mod);
|
|
4925
5682
|
} catch {
|
|
4926
|
-
_streetAbbrevs
|
|
5683
|
+
_streetAbbrevs ??= /* @__PURE__ */ new Set();
|
|
4927
5684
|
}
|
|
4928
5685
|
};
|
|
4929
5686
|
/** Ensure street abbreviation data is loaded. */
|
|
@@ -5035,7 +5792,7 @@ const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
|
5035
5792
|
}
|
|
5036
5793
|
return results;
|
|
5037
5794
|
};
|
|
5038
|
-
const ORPHAN_STREET_RE =
|
|
5795
|
+
const ORPHAN_STREET_RE = /^[^\S\n]*(\p{Lu}[\p{Ll}\p{Lu}]+(?:[^\S\n]+[\p{Lu}\p{Ll}][\p{Ll}]+)*[^\S\n]+\d{2,4}[a-zA-Z]?)[^\S\n]*$/gmu;
|
|
5039
5796
|
/**
|
|
5040
5797
|
* In the header zone (top 15%), find standalone lines
|
|
5041
5798
|
* matching "[Uppercase word(s)] [number]" that sit
|
|
@@ -5413,6 +6170,7 @@ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-lis
|
|
|
5413
6170
|
* of `\s` to avoid merging entities across newlines.
|
|
5414
6171
|
*/
|
|
5415
6172
|
const GAP_PATTERN = /^[ \t,\-]+$/;
|
|
6173
|
+
const isLegalFormOrganization = (entity) => entity.label === "organization" && entity.source === DETECTION_SOURCES.LEGAL_FORM;
|
|
5416
6174
|
/**
|
|
5417
6175
|
* Build a set of word boundary offsets for the full
|
|
5418
6176
|
* text using `Intl.Segmenter`. Returns a sorted array
|
|
@@ -5552,7 +6310,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
5552
6310
|
break;
|
|
5553
6311
|
}
|
|
5554
6312
|
}
|
|
5555
|
-
if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
6313
|
+
if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev) && isLegalFormOrganization(entity) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
5556
6314
|
prev.end = entity.end;
|
|
5557
6315
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
5558
6316
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -5710,9 +6468,12 @@ const enforceBoundaryConsistency = (entities, fullText) => {
|
|
|
5710
6468
|
//#region src/build-unified-search.ts
|
|
5711
6469
|
const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
|
|
5712
6470
|
const ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
6471
|
+
const createAllowedLabelSet$1 = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
6472
|
+
const labelIsAllowed$1 = (label, allowedLabels) => allowedLabels === null || allowedLabels.has(label);
|
|
5713
6473
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
5714
6474
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
5715
|
-
const
|
|
6475
|
+
const allowedLabels = createAllowedLabelSet$1(config.enableHotwordRules === true ? expandLabelsForHotwordRules(config.labels) : config.labels);
|
|
6476
|
+
const customRegexes = config.enableRegex ? (config.customRegexes ?? []).filter((entry) => labelIsAllowed$1(entry.label, allowedLabels)) : [];
|
|
5716
6477
|
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
5717
6478
|
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
5718
6479
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
@@ -5721,22 +6482,30 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
5721
6482
|
}),
|
|
5722
6483
|
config.enableDenyList ? buildDenyList(config, ctx) : Promise.resolve(null),
|
|
5723
6484
|
buildStreetTypePatterns(),
|
|
5724
|
-
getCurrencyPatterns(),
|
|
5725
|
-
getDatePatterns(),
|
|
5726
|
-
getSigningClausePatterns()
|
|
6485
|
+
config.enableRegex && labelIsAllowed$1("monetary amount", allowedLabels) ? getCurrencyPatterns() : Promise.resolve([]),
|
|
6486
|
+
config.enableRegex && labelIsAllowed$1("date", allowedLabels) ? getDatePatterns() : Promise.resolve([]),
|
|
6487
|
+
config.enableRegex && labelIsAllowed$1("address", allowedLabels) ? getSigningClausePatterns() : Promise.resolve([])
|
|
5727
6488
|
]);
|
|
5728
|
-
const allRegex = [
|
|
5729
|
-
|
|
5730
|
-
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
|
|
5739
|
-
|
|
6489
|
+
const allRegex = [];
|
|
6490
|
+
const regexMeta = [];
|
|
6491
|
+
if (config.enableRegex) for (const [index, pattern] of REGEX_PATTERNS.entries()) {
|
|
6492
|
+
const meta = REGEX_META[index];
|
|
6493
|
+
if (!meta || !labelIsAllowed$1(meta.label, allowedLabels)) continue;
|
|
6494
|
+
allRegex.push(pattern);
|
|
6495
|
+
regexMeta.push(meta);
|
|
6496
|
+
}
|
|
6497
|
+
for (const pattern of currencyPatterns) {
|
|
6498
|
+
allRegex.push(pattern);
|
|
6499
|
+
regexMeta.push(CURRENCY_PATTERN_META);
|
|
6500
|
+
}
|
|
6501
|
+
for (const pattern of datePatterns) {
|
|
6502
|
+
allRegex.push(pattern);
|
|
6503
|
+
regexMeta.push(DATE_PATTERN_META);
|
|
6504
|
+
}
|
|
6505
|
+
for (const pattern of signingPatterns) {
|
|
6506
|
+
allRegex.push(pattern);
|
|
6507
|
+
regexMeta.push(SIGNING_CLAUSE_META);
|
|
6508
|
+
}
|
|
5740
6509
|
const customRegexMeta = customRegexes.map((entry) => ({
|
|
5741
6510
|
label: entry.label,
|
|
5742
6511
|
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
|
|
@@ -5952,6 +6721,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
5952
6721
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
5953
6722
|
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
5954
6723
|
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
6724
|
+
const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
|
|
6725
|
+
const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
|
|
5955
6726
|
const shouldReplace = (a, b) => {
|
|
5956
6727
|
const aLen = a.end - a.start;
|
|
5957
6728
|
const bLen = b.end - b.start;
|
|
@@ -5967,6 +6738,31 @@ const shouldReplace = (a, b) => {
|
|
|
5967
6738
|
/** Labels where colons are structurally significant. */
|
|
5968
6739
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
5969
6740
|
/**
|
|
6741
|
+
* Labels whose entities should have a trailing sentence
|
|
6742
|
+
* `.` stripped during sanitisation. Restricted to
|
|
6743
|
+
* proper-noun-style labels where a final period is
|
|
6744
|
+
* almost always the sentence terminator that ran into
|
|
6745
|
+
* the capture, not a structural part of the value.
|
|
6746
|
+
* Numeric labels (`date`, `date of birth`, `phone
|
|
6747
|
+
* number`, `monetary amount`, `time`) and `person`
|
|
6748
|
+
* stay out — German writes `21. März`, post-nominal
|
|
6749
|
+
* degrees write `M.Sc.`, times write `5:00 p.m.`, and
|
|
6750
|
+
* stripping the dot would corrupt those spans.
|
|
6751
|
+
*/
|
|
6752
|
+
const PERIOD_STRIPPED_LABELS = new Set([
|
|
6753
|
+
"organization",
|
|
6754
|
+
"location",
|
|
6755
|
+
"address"
|
|
6756
|
+
]);
|
|
6757
|
+
const ADDRESS_FINAL_TOKEN_RE = /(?:^|[\s,])([\p{L}\p{M}.]+\.)$/u;
|
|
6758
|
+
const LOCATION_FINAL_DOTTED_ABBREV_RE = /(?:^|[\s,])(?:\p{Lu}\.){2,}$/u;
|
|
6759
|
+
const hasKnownAddressFinalAbbrev = (text) => {
|
|
6760
|
+
const finalToken = ADDRESS_FINAL_TOKEN_RE.exec(text)?.[1];
|
|
6761
|
+
if (!finalToken) return false;
|
|
6762
|
+
return getStreetAbbrevs().has(finalToken.toLowerCase());
|
|
6763
|
+
};
|
|
6764
|
+
const hasLocationFinalAbbrev = (text) => LOCATION_FINAL_DOTTED_ABBREV_RE.test(text);
|
|
6765
|
+
/**
|
|
5970
6766
|
* Labels whose detectors emit precise, evidence-backed spans. When
|
|
5971
6767
|
* one of these fires at the exact same offsets as a fuzzier
|
|
5972
6768
|
* `address` hit (city dictionary lookup, address-seed cluster), the
|
|
@@ -6039,16 +6835,63 @@ const resolveSameSpanLabelConflicts = (entities) => {
|
|
|
6039
6835
|
if (dropped.size === 0) return entities;
|
|
6040
6836
|
return entities.filter((e) => !dropped.has(e));
|
|
6041
6837
|
};
|
|
6838
|
+
/**
|
|
6839
|
+
* Trailing typographic punctuation that detectors
|
|
6840
|
+
* occasionally swallow when a capture runs to the end
|
|
6841
|
+
* of a sentence or quoted phrase. Stripped from every
|
|
6842
|
+
* non-literal, non-locked entity. Curated dictionary and
|
|
6843
|
+
* gazetteer entries with punctuation that is clearly part of
|
|
6844
|
+
* the literal (`Hello bank!`, `"Juez y parte"`) keep their
|
|
6845
|
+
* own boundaries. Generated/extended spans from the same
|
|
6846
|
+
* sources still pass through cleanup so dangling punctuation
|
|
6847
|
+
* does not become part of the redaction
|
|
6848
|
+
* (e.g. `Bond Hedge Documentation"` →
|
|
6849
|
+
* `Bond Hedge Documentation`).
|
|
6850
|
+
*
|
|
6851
|
+
* `)` is deliberately omitted — monetary amounts are
|
|
6852
|
+
* extended to include trailing "(slovy ...)" / "(in
|
|
6853
|
+
* words ...)" parentheticals where the closing paren
|
|
6854
|
+
* is structural, and stripping it would leave the open
|
|
6855
|
+
* paren dangling. `.` is also omitted because the
|
|
6856
|
+
* trailing-period rule below has label-aware handling
|
|
6857
|
+
* (legal-form abbreviations keep their dot).
|
|
6858
|
+
*/
|
|
6859
|
+
const TRAILING_PUNCT_CLASS = `["“”‘’'»!?]`;
|
|
6860
|
+
/**
|
|
6861
|
+
* Leading typographic punctuation that detectors
|
|
6862
|
+
* occasionally swallow when a capture starts at an
|
|
6863
|
+
* opening quote. `(` is deliberately omitted — it is
|
|
6864
|
+
* almost always the opening of a structural
|
|
6865
|
+
* parenthetical (registration number group, monetary
|
|
6866
|
+
* "(slovy ...)" extension) that the detector
|
|
6867
|
+
* intentionally captured.
|
|
6868
|
+
*/
|
|
6869
|
+
const LEADING_PUNCT_CLASS = `["“”‘’'«¿¡]`;
|
|
6870
|
+
const STRIP_BY_LABEL = {
|
|
6871
|
+
colon: /[\s,;]+/,
|
|
6872
|
+
default: /[\s:,;]+/
|
|
6873
|
+
};
|
|
6874
|
+
const LEADING_TRIM_BY_LABEL = {
|
|
6875
|
+
colon: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.colon.source}|${LEADING_PUNCT_CLASS})+`),
|
|
6876
|
+
default: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.default.source}|${LEADING_PUNCT_CLASS})+`)
|
|
6877
|
+
};
|
|
6878
|
+
const TRAILING_TRIM_BY_LABEL = {
|
|
6879
|
+
colon: new RegExp(`(?:${STRIP_BY_LABEL.colon.source}|${TRAILING_PUNCT_CLASS})+$`),
|
|
6880
|
+
default: new RegExp(`(?:${STRIP_BY_LABEL.default.source}|${TRAILING_PUNCT_CLASS})+$`)
|
|
6881
|
+
};
|
|
6042
6882
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
6043
6883
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
6044
|
-
if (hasLockedBoundary(e)) return [e];
|
|
6045
|
-
const
|
|
6046
|
-
const
|
|
6884
|
+
if (hasLockedBoundary(e) || hasCuratedLiteralBoundary(e)) return [e];
|
|
6885
|
+
const stripKind = COLON_LABELS.has(e.label) ? "colon" : "default";
|
|
6886
|
+
const leadRe = LEADING_TRIM_BY_LABEL[stripKind];
|
|
6887
|
+
const trailRe = TRAILING_TRIM_BY_LABEL[stripKind];
|
|
6888
|
+
const leadTrimmed = e.text.replace(leadRe, "");
|
|
6047
6889
|
const lead = e.text.length - leadTrimmed.length;
|
|
6048
|
-
let cleaned = leadTrimmed.replace(
|
|
6049
|
-
if (e.label
|
|
6050
|
-
if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6890
|
+
let cleaned = leadTrimmed.replace(trailRe, "");
|
|
6891
|
+
if (PERIOD_STRIPPED_LABELS.has(e.label) && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
6892
|
+
if (!(getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix)) || e.label === "address" && hasKnownAddressFinalAbbrev(cleaned) || e.label === "location" && hasLocationFinalAbbrev(cleaned))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6051
6893
|
}
|
|
6894
|
+
cleaned = cleaned.replace(trailRe, "");
|
|
6052
6895
|
if (cleaned.length === 0) return [];
|
|
6053
6896
|
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
6054
6897
|
const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
|
|
@@ -6105,7 +6948,7 @@ let amountWordsLoaded = false;
|
|
|
6105
6948
|
const getAmountWordsRe = async () => {
|
|
6106
6949
|
if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
|
|
6107
6950
|
try {
|
|
6108
|
-
const alt = (await import("./amount-words.mjs")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
6951
|
+
const alt = (await import("./amount-words.mjs").then((n) => n.n)).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
6109
6952
|
amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
|
|
6110
6953
|
} catch {
|
|
6111
6954
|
amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
|
|
@@ -6154,7 +6997,7 @@ const configKey = (config, gazetteerEntries) => {
|
|
|
6154
6997
|
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
|
|
6155
6998
|
})).sort().join("\n") : "";
|
|
6156
6999
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
6157
|
-
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
7000
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.nameCorpusLanguages?.toSorted().join(",") ?? ""}:${config.enableRegex}:${config.labels.toSorted().join(",")}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
6158
7001
|
};
|
|
6159
7002
|
/**
|
|
6160
7003
|
* Get or build a cached search instance. Cache state
|
|
@@ -6174,6 +7017,13 @@ const getCachedSearch = async (config, gazetteerEntries, ctx) => {
|
|
|
6174
7017
|
return result;
|
|
6175
7018
|
};
|
|
6176
7019
|
/**
|
|
7020
|
+
* Pre-build and cache the unified search instance for a
|
|
7021
|
+
* pipeline configuration. Use the same context in
|
|
7022
|
+
* `runPipeline` to reuse the prepared automata without
|
|
7023
|
+
* passing `cachedSearch` around manually.
|
|
7024
|
+
*/
|
|
7025
|
+
const preparePipelineSearch = ({ config, gazetteerEntries = [], context }) => getCachedSearch(config, gazetteerEntries, context ?? defaultContext);
|
|
7026
|
+
/**
|
|
6177
7027
|
* Run the full detection pipeline.
|
|
6178
7028
|
*
|
|
6179
7029
|
* Two TextSearch instances scan the text (regex +
|
|
@@ -6230,7 +7080,7 @@ const runPipeline = async (options) => {
|
|
|
6230
7080
|
warmAddressStopKeywords(),
|
|
6231
7081
|
hotwordInit
|
|
6232
7082
|
]);
|
|
6233
|
-
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
7083
|
+
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
6234
7084
|
let zones = [];
|
|
6235
7085
|
if (config.enableZoneClassification && zoneInitOk) {
|
|
6236
7086
|
zones = classifyZones(fullText, ctx);
|
|
@@ -6239,7 +7089,11 @@ const runPipeline = async (options) => {
|
|
|
6239
7089
|
checkAbort(signal);
|
|
6240
7090
|
const hotwordsActive = enableHotwords && hotwordInitOk;
|
|
6241
7091
|
const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
|
|
6242
|
-
const
|
|
7092
|
+
const searchConfig = hotwordsActive ? {
|
|
7093
|
+
...config,
|
|
7094
|
+
labels: [...expandLabelsForHotwordRules(config.labels)]
|
|
7095
|
+
} : config;
|
|
7096
|
+
const search = cachedSearch ?? await getCachedSearch(searchConfig, gazetteerEntries, ctx);
|
|
6243
7097
|
checkAbort(signal);
|
|
6244
7098
|
const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
6245
7099
|
const { slices } = search;
|
|
@@ -6259,7 +7113,7 @@ const runPipeline = async (options) => {
|
|
|
6259
7113
|
let rawNameCorpusEntities = [];
|
|
6260
7114
|
let nameCorpusEntities = [];
|
|
6261
7115
|
if (config.enableNameCorpus && !config.enableDenyList) {
|
|
6262
|
-
await initNameCorpus(ctx, config.dictionaries);
|
|
7116
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
6263
7117
|
checkAbort(signal);
|
|
6264
7118
|
rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
|
|
6265
7119
|
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|
|
@@ -6958,6 +7812,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
6958
7812
|
//#region src/wasm.ts
|
|
6959
7813
|
initTextSearch(TextSearch);
|
|
6960
7814
|
//#endregion
|
|
6961
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
7815
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
6962
7816
|
|
|
6963
7817
|
//# sourceMappingURL=wasm.mjs.map
|