@stll/anonymize-wasm 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-street-types.mjs +20 -1
- package/dist/allow-list.mjs +12 -0
- package/dist/amount-words.mjs +472 -36
- package/dist/generic-roles.mjs +1 -0
- package/dist/rolldown-runtime.mjs +13 -0
- package/dist/structural-single-cap-prefixes.mjs +99 -0
- package/dist/structural-single-cap-prefixes.mjs.map +1 -0
- package/dist/triggers.en.mjs +16 -1
- package/dist/wasm.d.mts +11 -1
- package/dist/wasm.mjs +712 -102
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
|
|
2
|
+
import { t as amount_words_default } from "./amount-words.mjs";
|
|
3
|
+
import { t as address_street_types_default } from "./address-street-types.mjs";
|
|
2
4
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
3
5
|
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
4
6
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
@@ -885,6 +887,7 @@ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTE
|
|
|
885
887
|
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
886
888
|
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
887
889
|
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
890
|
+
const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
|
|
888
891
|
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
889
892
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
890
893
|
let legalRoleHeadsCache = null;
|
|
@@ -912,11 +915,22 @@ const warmLegalRoleHeads = async () => {
|
|
|
912
915
|
loadLegalRoleHeads(),
|
|
913
916
|
loadAllLegalSuffixes(),
|
|
914
917
|
loadSentenceVerbIndicators(),
|
|
915
|
-
loadClauseNounHeads()
|
|
918
|
+
loadClauseNounHeads(),
|
|
919
|
+
loadConnectorProseHeads(),
|
|
920
|
+
loadStructuralSingleCapPrefixes()
|
|
916
921
|
]);
|
|
917
922
|
};
|
|
918
923
|
let allLegalSuffixesCache = null;
|
|
919
924
|
let allLegalSuffixesPromise = null;
|
|
925
|
+
let normalizedLegalBoundarySuffixesCache = null;
|
|
926
|
+
let normalizedInNameLegalFormWordsCache = null;
|
|
927
|
+
const normalizeLegalSuffixToken = (suffix) => suffix.replace(/[.,\s]/g, "");
|
|
928
|
+
const isBoundaryLegalSuffixForm = (form) => {
|
|
929
|
+
const normalized = normalizeLegalSuffixToken(form);
|
|
930
|
+
if (normalized.length === 0) return false;
|
|
931
|
+
if (LEGAL_SUFFIXES.includes(form)) return true;
|
|
932
|
+
return /[.]/u.test(form) || normalized === normalized.toUpperCase();
|
|
933
|
+
};
|
|
920
934
|
const loadAllLegalSuffixes = async () => {
|
|
921
935
|
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
922
936
|
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
@@ -941,11 +955,15 @@ const loadAllLegalSuffixes = async () => {
|
|
|
941
955
|
}
|
|
942
956
|
out.sort((a, b) => b.length - a.length);
|
|
943
957
|
allLegalSuffixesCache = out;
|
|
958
|
+
normalizedLegalBoundarySuffixesCache = new Set(out.filter(isBoundaryLegalSuffixForm).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
959
|
+
normalizedInNameLegalFormWordsCache = new Set(out.filter((form) => !isBoundaryLegalSuffixForm(form) && !/\s/u.test(form)).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
944
960
|
return out;
|
|
945
961
|
})();
|
|
946
962
|
return allLegalSuffixesPromise;
|
|
947
963
|
};
|
|
948
964
|
const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
|
|
965
|
+
const getNormalizedLegalBoundarySuffixesSync = () => normalizedLegalBoundarySuffixesCache ?? new Set(LEGAL_SUFFIXES.map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
966
|
+
const getNormalizedInNameLegalFormWordsSync = () => normalizedInNameLegalFormWordsCache ?? /* @__PURE__ */ new Set();
|
|
949
967
|
/**
|
|
950
968
|
* Sync accessor for the full legal-form vocabulary
|
|
951
969
|
* (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
|
|
@@ -985,18 +1003,72 @@ const loadClauseNounHeads = async () => {
|
|
|
985
1003
|
return clauseNounHeadsPromise;
|
|
986
1004
|
};
|
|
987
1005
|
const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
|
|
1006
|
+
let connectorProseHeadsCache = null;
|
|
1007
|
+
let connectorProseHeadsPromise = null;
|
|
1008
|
+
const loadConnectorProseHeads = async () => {
|
|
1009
|
+
if (connectorProseHeadsCache) return connectorProseHeadsCache;
|
|
1010
|
+
if (connectorProseHeadsPromise) return connectorProseHeadsPromise;
|
|
1011
|
+
connectorProseHeadsPromise = (async () => {
|
|
1012
|
+
let data = {};
|
|
1013
|
+
try {
|
|
1014
|
+
const mod = await import("./generic-roles.mjs");
|
|
1015
|
+
data = mod.default ?? mod;
|
|
1016
|
+
} catch (err) {
|
|
1017
|
+
console.warn("[anonymize] legal-forms: failed to load generic-roles.json:", err);
|
|
1018
|
+
}
|
|
1019
|
+
const all = /* @__PURE__ */ new Set();
|
|
1020
|
+
if (Array.isArray(data.roles)) {
|
|
1021
|
+
for (const role of data.roles) if (typeof role === "string" && role.length > 0) all.add(role.toLowerCase());
|
|
1022
|
+
}
|
|
1023
|
+
connectorProseHeadsCache = all;
|
|
1024
|
+
return all;
|
|
1025
|
+
})();
|
|
1026
|
+
return connectorProseHeadsPromise;
|
|
1027
|
+
};
|
|
1028
|
+
const getConnectorProseHeadsSync = () => connectorProseHeadsCache ?? /* @__PURE__ */ new Set();
|
|
1029
|
+
let structuralSingleCapPrefixesCache = null;
|
|
1030
|
+
let structuralSingleCapPrefixesPromise = null;
|
|
1031
|
+
const loadStructuralSingleCapPrefixes = async () => {
|
|
1032
|
+
if (structuralSingleCapPrefixesCache) return structuralSingleCapPrefixesCache;
|
|
1033
|
+
if (structuralSingleCapPrefixesPromise) return structuralSingleCapPrefixesPromise;
|
|
1034
|
+
structuralSingleCapPrefixesPromise = (async () => {
|
|
1035
|
+
let data = {};
|
|
1036
|
+
try {
|
|
1037
|
+
const mod = await import("./structural-single-cap-prefixes.mjs");
|
|
1038
|
+
data = mod.default ?? mod;
|
|
1039
|
+
} catch (err) {
|
|
1040
|
+
console.warn("[anonymize] legal-forms: failed to load structural-single-cap-prefixes.json:", err);
|
|
1041
|
+
}
|
|
1042
|
+
const all = /* @__PURE__ */ new Set();
|
|
1043
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1044
|
+
if (key.startsWith("_")) continue;
|
|
1045
|
+
if (!Array.isArray(value)) continue;
|
|
1046
|
+
for (const prefix of value) {
|
|
1047
|
+
if (typeof prefix !== "string" || prefix.length === 0) continue;
|
|
1048
|
+
all.add(prefix.toLowerCase());
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
structuralSingleCapPrefixesCache = all;
|
|
1052
|
+
return all;
|
|
1053
|
+
})();
|
|
1054
|
+
return structuralSingleCapPrefixesPromise;
|
|
1055
|
+
};
|
|
1056
|
+
const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
|
|
988
1057
|
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
989
1058
|
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1059
|
+
const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
990
1060
|
const buildPatternString = (forms) => {
|
|
991
1061
|
if (forms.length === 0) return null;
|
|
992
1062
|
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
993
1063
|
const HSPACE = "[^\\S\\n]";
|
|
994
1064
|
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
995
1065
|
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
996
|
-
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
|
|
1066
|
+
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
|
|
997
1067
|
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
998
1068
|
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
999
|
-
|
|
1069
|
+
const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
|
|
1070
|
+
const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
|
|
1071
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
1000
1072
|
};
|
|
1001
1073
|
/**
|
|
1002
1074
|
* Build legal form regex pattern strings.
|
|
@@ -1029,6 +1101,7 @@ const buildLegalFormPatterns = async () => {
|
|
|
1029
1101
|
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1030
1102
|
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1031
1103
|
patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1104
|
+
patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
|
|
1032
1105
|
return patterns;
|
|
1033
1106
|
};
|
|
1034
1107
|
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
@@ -1038,6 +1111,12 @@ const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Ho
|
|
|
1038
1111
|
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1039
1112
|
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1040
1113
|
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
1114
|
+
const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:[ \\t]+|,[ \\t]*)`, "u");
|
|
1115
|
+
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
|
|
1116
|
+
const isStructuralSingleCapMatch = (text) => {
|
|
1117
|
+
const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
|
|
1118
|
+
return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
|
|
1119
|
+
};
|
|
1041
1120
|
/**
|
|
1042
1121
|
* Find the word ending just before `pos` in `text`,
|
|
1043
1122
|
* skipping any whitespace (not newlines).
|
|
@@ -1062,25 +1141,145 @@ const findWordBefore = (text, pos) => {
|
|
|
1062
1141
|
start: wordStart
|
|
1063
1142
|
};
|
|
1064
1143
|
};
|
|
1144
|
+
const hasSingleCapPrefixBefore = (fullText, matchStart) => {
|
|
1145
|
+
const prev = findWordBefore(fullText, matchStart);
|
|
1146
|
+
return prev !== null && prev.word.length === 1 && UPPER_LETTER_RE.test(prev.word);
|
|
1147
|
+
};
|
|
1148
|
+
const isBareSingleCapStructuralInnerMatch = (fullText, matchStart, text) => {
|
|
1149
|
+
if (!BARE_SINGLE_CAP_LEGAL_FORM_RE.test(text)) return false;
|
|
1150
|
+
const prev = findWordBefore(fullText, matchStart);
|
|
1151
|
+
return prev !== null && getStructuralSingleCapPrefixesSync().has(prev.word.toLowerCase());
|
|
1152
|
+
};
|
|
1153
|
+
const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
|
|
1154
|
+
let cut = -1;
|
|
1155
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1156
|
+
const suffixClean = suffix.replace(/[.,\s]/g, "");
|
|
1157
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1158
|
+
let fromIndex = 0;
|
|
1159
|
+
while (fromIndex < entityText.length) {
|
|
1160
|
+
const suffixStart = entityText.indexOf(suffix, fromIndex);
|
|
1161
|
+
if (suffixStart === -1) break;
|
|
1162
|
+
fromIndex = suffixStart + suffix.length;
|
|
1163
|
+
const suffixEnd = suffixStart + suffix.length;
|
|
1164
|
+
if (suffixEnd >= entityText.length - 1) continue;
|
|
1165
|
+
const afterSuffix = entityText.slice(suffixEnd);
|
|
1166
|
+
const boundary = /^,\s+(?=\p{Lu})/u.exec(afterSuffix);
|
|
1167
|
+
if (boundary === null) continue;
|
|
1168
|
+
const nextStart = suffixEnd + boundary[0].length;
|
|
1169
|
+
const remainder = entityText.slice(nextStart);
|
|
1170
|
+
if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
|
|
1171
|
+
cut = Math.max(cut, nextStart);
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
if (cut <= 0) return {
|
|
1175
|
+
entityStart,
|
|
1176
|
+
entityText
|
|
1177
|
+
};
|
|
1178
|
+
return {
|
|
1179
|
+
entityStart: entityStart + cut,
|
|
1180
|
+
entityText: entityText.slice(cut)
|
|
1181
|
+
};
|
|
1182
|
+
};
|
|
1183
|
+
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
1184
|
+
const previousWord = findWordBefore(fullText, pos);
|
|
1185
|
+
if (!previousWord) return false;
|
|
1186
|
+
let scan = previousWord.start - 1;
|
|
1187
|
+
while (scan >= 0 && (fullText[scan] === " " || fullText[scan] === " ")) scan--;
|
|
1188
|
+
return scan >= 1 && fullText[scan] === "." && UPPER_LETTER_RE.test(fullText[scan - 1] ?? "");
|
|
1189
|
+
};
|
|
1065
1190
|
/**
|
|
1066
1191
|
* Count consecutive uppercase-starting words immediately
|
|
1067
1192
|
* before `pos`. Stops at the first non-upper word or at
|
|
1068
|
-
* text/line start. Used to disambiguate
|
|
1069
|
-
* and <ORG>"
|
|
1070
|
-
|
|
1071
|
-
|
|
1193
|
+
* text/line start. Used to disambiguate sentence prose
|
|
1194
|
+
* ("<First> <Last> and <ORG>", "<Defined-Term> and
|
|
1195
|
+
* <ORG>") from multi-word organisation names that span
|
|
1196
|
+
* an "and" connector ("UniCredit Bank Czech Republic and
|
|
1197
|
+
* Slovakia, a.s.").
|
|
1198
|
+
*
|
|
1199
|
+
* When `crossInNamePreps` is true, the walk also steps
|
|
1200
|
+
* over in-name lowercase prepositions ("of", "the") as
|
|
1201
|
+
* long as they sit between two upper words. This lets
|
|
1202
|
+
* the suffix-mode "and"-crossing logic see through
|
|
1203
|
+
* "<Trust ← and ← America ← of ← Bank>" and emit one
|
|
1204
|
+
* full organisation span.
|
|
1205
|
+
*/
|
|
1206
|
+
const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
|
|
1072
1207
|
let count = 0;
|
|
1073
1208
|
let scan = pos;
|
|
1074
1209
|
while (scan > 0) {
|
|
1075
1210
|
const found = findWordBefore(fullText, scan);
|
|
1076
|
-
if (
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1211
|
+
if (found) {
|
|
1212
|
+
if (UPPER_LETTER_RE.test(found.word)) {
|
|
1213
|
+
count++;
|
|
1214
|
+
scan = found.start;
|
|
1215
|
+
continue;
|
|
1216
|
+
}
|
|
1217
|
+
if (crossInNamePreps && IN_NAME_PREPOSITION_RE.test(found.word)) {
|
|
1218
|
+
const prev = findWordBefore(fullText, found.start);
|
|
1219
|
+
if (!prev) break;
|
|
1220
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1221
|
+
scan = found.start;
|
|
1222
|
+
continue;
|
|
1223
|
+
}
|
|
1224
|
+
break;
|
|
1225
|
+
}
|
|
1226
|
+
let p = scan - 1;
|
|
1227
|
+
while (p >= 0 && (fullText[p] === " " || fullText[p] === " ")) p--;
|
|
1228
|
+
if (p >= 1 && fullText[p] === "." && UPPER_LETTER_RE.test(fullText[p - 1] ?? "")) {
|
|
1229
|
+
count++;
|
|
1230
|
+
scan = p - 1;
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
break;
|
|
1080
1234
|
}
|
|
1081
1235
|
return count;
|
|
1082
1236
|
};
|
|
1083
1237
|
/**
|
|
1238
|
+
* True when `word` is a recognized legal-form suffix
|
|
1239
|
+
* (case-sensitive against the legal-forms vocabulary).
|
|
1240
|
+
* Used when deciding whether to cross an "and" connector
|
|
1241
|
+
* during backward extension — if the word immediately
|
|
1242
|
+
* preceding the connector is itself a legal-form suffix,
|
|
1243
|
+
* the "and" sits between two organisation names rather
|
|
1244
|
+
* than inside one ("Morgan Securities LLC and Allen &
|
|
1245
|
+
* Company LLC"), so the walk must stop there.
|
|
1246
|
+
*/
|
|
1247
|
+
const isKnownLegalFormSuffix = (word) => {
|
|
1248
|
+
if (word.length === 0) return false;
|
|
1249
|
+
return getNormalizedLegalBoundarySuffixesSync().has(word);
|
|
1250
|
+
};
|
|
1251
|
+
const isInNameLegalFormWord = (word) => {
|
|
1252
|
+
if (word.length === 0) return false;
|
|
1253
|
+
return getNormalizedInNameLegalFormWordsSync().has(word);
|
|
1254
|
+
};
|
|
1255
|
+
/**
|
|
1256
|
+
* If `pos` is immediately preceded (modulo horizontal
|
|
1257
|
+
* whitespace) by an initial-dot run like `J.P.`, `U.S.`,
|
|
1258
|
+
* or `N.A.`, return the position where the initial run
|
|
1259
|
+
* starts. Otherwise return `pos` unchanged. The run must
|
|
1260
|
+
* be word-bounded on the left so we never absorb a stray
|
|
1261
|
+
* sentence-ending dot.
|
|
1262
|
+
*/
|
|
1263
|
+
const skipInitialsBackward = (fullText, pos) => {
|
|
1264
|
+
let scan = pos - 1;
|
|
1265
|
+
while (scan >= 0) {
|
|
1266
|
+
const ch = fullText.charAt(scan);
|
|
1267
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1268
|
+
scan--;
|
|
1269
|
+
}
|
|
1270
|
+
if (scan < 0 || fullText.charAt(scan) !== ".") return pos;
|
|
1271
|
+
const scanLimit = Math.max(0, scan + 1 - 100);
|
|
1272
|
+
const head = fullText.slice(scanLimit, scan + 1);
|
|
1273
|
+
const match = /(?:\p{Lu}\.[^\S\n]?){2,}$/u.exec(head);
|
|
1274
|
+
if (match === null) return pos;
|
|
1275
|
+
const start = scanLimit + match.index;
|
|
1276
|
+
if (start > 0) {
|
|
1277
|
+
const prevCh = fullText.charAt(start - 1);
|
|
1278
|
+
if (/[\p{L}\p{M}\p{N}]/u.test(prevCh)) return pos;
|
|
1279
|
+
}
|
|
1280
|
+
return start;
|
|
1281
|
+
};
|
|
1282
|
+
/**
|
|
1084
1283
|
* Extend a match backward through uppercase words and
|
|
1085
1284
|
* lowercase connectors. Stops at start of text,
|
|
1086
1285
|
* newline, or a word that doesn't qualify.
|
|
@@ -1109,19 +1308,30 @@ const extendBackward = (fullText, matchStart, options = {}) => {
|
|
|
1109
1308
|
const isConnector = CONNECTOR_RE.test(word);
|
|
1110
1309
|
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1111
1310
|
if (isUpper) pos = wordStart;
|
|
1112
|
-
else if (isConnector) {
|
|
1113
|
-
|
|
1311
|
+
else if (isConnector) if (AND_TYPE_CONNECTOR_RE.test(word)) {
|
|
1312
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1313
|
+
if (!prev) break;
|
|
1314
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1315
|
+
if (isKnownLegalFormSuffix(prev.word)) break;
|
|
1316
|
+
const upperWordsBefore = countUpperWordsBefore(fullText, wordStart, suffixMode);
|
|
1317
|
+
const middleInitialBefore = hasMiddleInitialBefore(fullText, wordStart);
|
|
1318
|
+
if (upperWordsBefore <= 1 && (getClauseNounHeadsSync().has(prev.word.toLowerCase()) || getConnectorProseHeadsSync().has(prev.word.toLowerCase()))) break;
|
|
1319
|
+
if (suffixMode ? middleInitialBefore && hasSingleCapPrefixBefore(fullText, matchStart) : upperWordsBefore === 2 && !isInNameLegalFormWord(prev.word) || middleInitialBefore) break;
|
|
1320
|
+
pos = prev.start;
|
|
1321
|
+
} else {
|
|
1114
1322
|
const prev = findWordBefore(fullText, wordStart);
|
|
1115
1323
|
if (!prev) break;
|
|
1116
1324
|
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1117
1325
|
pos = prev.start;
|
|
1118
|
-
}
|
|
1326
|
+
}
|
|
1327
|
+
else if (isInNamePrep) {
|
|
1119
1328
|
const prev = findWordBefore(fullText, wordStart);
|
|
1120
1329
|
if (!prev) break;
|
|
1121
1330
|
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1122
1331
|
pos = prev.start;
|
|
1123
1332
|
} else break;
|
|
1124
1333
|
}
|
|
1334
|
+
pos = skipInitialsBackward(fullText, pos);
|
|
1125
1335
|
return pos;
|
|
1126
1336
|
};
|
|
1127
1337
|
const trimLeadingClause = (text) => {
|
|
@@ -1162,6 +1372,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1162
1372
|
const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
|
|
1163
1373
|
let processedStart = match.start;
|
|
1164
1374
|
let processedText = text;
|
|
1375
|
+
if (isStructuralSingleCapMatch(processedText) || fullText !== void 0 && isBareSingleCapStructuralInnerMatch(fullText, match.start, text)) continue;
|
|
1165
1376
|
let trimmed = false;
|
|
1166
1377
|
const firstWordText = firstWordMatch?.[0] ?? "";
|
|
1167
1378
|
const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
|
|
@@ -1212,12 +1423,15 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1212
1423
|
let entityStart = processedStart;
|
|
1213
1424
|
let entityText = processedText;
|
|
1214
1425
|
if (fullText && !trimmed) {
|
|
1215
|
-
const extended = extendBackward(fullText, processedStart);
|
|
1426
|
+
const extended = !BARE_SINGLE_CAP_LEGAL_FORM_RE.test(processedText) ? extendBackward(fullText, processedStart) : processedStart;
|
|
1216
1427
|
if (extended < processedStart) {
|
|
1217
1428
|
entityStart = extended;
|
|
1218
1429
|
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1219
1430
|
}
|
|
1220
1431
|
}
|
|
1432
|
+
const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
|
|
1433
|
+
entityStart = listTrim.entityStart;
|
|
1434
|
+
entityText = listTrim.entityText;
|
|
1221
1435
|
const clauseTrim = trimLeadingClause(entityText);
|
|
1222
1436
|
if (clauseTrim.offset > 0) {
|
|
1223
1437
|
entityStart += clauseTrim.offset;
|
|
@@ -1564,20 +1778,23 @@ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, dat
|
|
|
1564
1778
|
const label = data.labels[localIdx];
|
|
1565
1779
|
if (!label) continue;
|
|
1566
1780
|
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
1781
|
+
const isExtended = extended !== null;
|
|
1567
1782
|
const end = extended?.end ?? match.end;
|
|
1568
1783
|
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
1569
1784
|
exactSpans.push({
|
|
1570
1785
|
start: match.start,
|
|
1571
1786
|
end
|
|
1572
1787
|
});
|
|
1573
|
-
|
|
1788
|
+
const entity = {
|
|
1574
1789
|
start: match.start,
|
|
1575
1790
|
end,
|
|
1576
1791
|
label,
|
|
1577
1792
|
text,
|
|
1578
1793
|
score: .9,
|
|
1579
1794
|
source: DETECTION_SOURCES.GAZETTEER
|
|
1580
|
-
}
|
|
1795
|
+
};
|
|
1796
|
+
if (isExtended) entity.sourceDetail = "gazetteer-extension";
|
|
1797
|
+
results.push(entity);
|
|
1581
1798
|
}
|
|
1582
1799
|
for (const match of allMatches) {
|
|
1583
1800
|
const idx = match.pattern;
|
|
@@ -2122,8 +2339,10 @@ const MIN_MONTH_NAME_LENGTH = 3;
|
|
|
2122
2339
|
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
2123
2340
|
/** Escape for use inside a regex alternation. */
|
|
2124
2341
|
const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2342
|
+
const escapeRegexPhrase = (s) => escapeRegex$2(s.trim()).replace(/\s+/g, "[^\\S\\n\\t]+");
|
|
2125
2343
|
/** Escape for use inside a regex character class. */
|
|
2126
2344
|
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
2345
|
+
const toSortedAlternation = (values) => [...new Set(values.map(escapeRegexPhrase).filter((value) => value.length > 0))].toSorted((a, b) => b.length - a.length).join("|");
|
|
2127
2346
|
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
2128
2347
|
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
2129
2348
|
const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
|
|
@@ -2134,6 +2353,7 @@ const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).ma
|
|
|
2134
2353
|
const escaped = escapeRegex$2(h);
|
|
2135
2354
|
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
2136
2355
|
}).join("|");
|
|
2356
|
+
const AMOUNT_WORDS = amount_words_default;
|
|
2137
2357
|
const toEntry = (validator, label, score) => {
|
|
2138
2358
|
const pattern = toRegex(validator).source;
|
|
2139
2359
|
if (!pattern) return null;
|
|
@@ -2347,6 +2567,58 @@ const SHORT_TLDS = "de|at|be|se|fi|dk|no|it|uk";
|
|
|
2347
2567
|
const HOST_LABEL = `[a-zA-Z0-9](?:[a-zA-Z0-9\\-]*[a-zA-Z0-9])?`;
|
|
2348
2568
|
const BARE_HOST = `\\b[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]`;
|
|
2349
2569
|
const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
|
|
2570
|
+
const BARE_DOMAIN = {
|
|
2571
|
+
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
2572
|
+
label: "url",
|
|
2573
|
+
score: .9
|
|
2574
|
+
};
|
|
2575
|
+
const IPV6_ADDRESS = {
|
|
2576
|
+
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
2577
|
+
label: "ip address",
|
|
2578
|
+
score: 1
|
|
2579
|
+
};
|
|
2580
|
+
const MAC_ADDRESS = {
|
|
2581
|
+
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
2582
|
+
label: "mac address",
|
|
2583
|
+
score: 1
|
|
2584
|
+
};
|
|
2585
|
+
const UK_POSTCODE = {
|
|
2586
|
+
pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
|
|
2587
|
+
label: "address",
|
|
2588
|
+
score: .9
|
|
2589
|
+
};
|
|
2590
|
+
const UK_NINO = {
|
|
2591
|
+
pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
|
|
2592
|
+
label: "social security number",
|
|
2593
|
+
score: .95,
|
|
2594
|
+
validator: gb.nino
|
|
2595
|
+
};
|
|
2596
|
+
const TIME_12H = {
|
|
2597
|
+
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
2598
|
+
label: "date",
|
|
2599
|
+
score: .9
|
|
2600
|
+
};
|
|
2601
|
+
const PERCENT_TOKEN = `${`(?:[+${DASH_INNER}])?(?:\\d{1,3}(?:[.,]\\d{3})+(?:[.,]\\d{1,4})?|\\d+(?:[.,]\\d{1,4})?)`}[^\\S\\n]{0,2}%`;
|
|
2602
|
+
const PERCENT_RANGE_NUMBER = `\\d+(?:[.,]\\d{1,4})?`;
|
|
2603
|
+
const PERCENT_RANGE = `${PERCENT_RANGE_NUMBER}[^\\S\\n]*${DASH}[^\\S\\n]*${PERCENT_RANGE_NUMBER}[^\\S\\n]{0,2}%`;
|
|
2604
|
+
const buildPercentWordPattern = (config) => {
|
|
2605
|
+
const phrases = [];
|
|
2606
|
+
for (const entry of config.percentages ?? []) {
|
|
2607
|
+
const ones = entry.ones.map(escapeRegex$2);
|
|
2608
|
+
const standalone = (entry.standalone ?? []).map(escapeRegex$2);
|
|
2609
|
+
const baseWords = [
|
|
2610
|
+
...ones,
|
|
2611
|
+
...entry.teens,
|
|
2612
|
+
...entry.tens
|
|
2613
|
+
].map(escapeRegex$2);
|
|
2614
|
+
const compoundSeparator = entry.allowSpaceCompoundSeparator ? `(?:${DASH}|[^\\S\\n]+)` : DASH;
|
|
2615
|
+
const compound = `(?:${baseWords.join("|")})` + (ones.length > 0 ? `(?:${compoundSeparator}(?:${ones.join("|")}))?` : "");
|
|
2616
|
+
const word = `(?:${[...standalone, compound].join("|")})`;
|
|
2617
|
+
const keyword = `(?:${entry.keywords.map(escapeRegex$2).join("|")})`;
|
|
2618
|
+
phrases.push(`${word}[^\\S\\n]+${keyword}`);
|
|
2619
|
+
}
|
|
2620
|
+
return phrases.length > 0 ? `(?i:(?:${phrases.join("|")}))` : "(?!)";
|
|
2621
|
+
};
|
|
2350
2622
|
/**
|
|
2351
2623
|
* All static PII regex definitions. Scanned in a
|
|
2352
2624
|
* single pass by @stll/regex-set (Rust DFA).
|
|
@@ -2388,36 +2660,16 @@ const ALL_REGEX_DEFS = [
|
|
|
2388
2660
|
BR_RG_WITH_SSP,
|
|
2389
2661
|
BR_OAB,
|
|
2390
2662
|
URL,
|
|
2663
|
+
IPV6_ADDRESS,
|
|
2664
|
+
MAC_ADDRESS,
|
|
2665
|
+
BARE_DOMAIN,
|
|
2666
|
+
UK_POSTCODE,
|
|
2667
|
+
UK_NINO,
|
|
2668
|
+
TIME_12H,
|
|
2391
2669
|
{
|
|
2392
|
-
pattern:
|
|
2393
|
-
label: "
|
|
2394
|
-
score:
|
|
2395
|
-
},
|
|
2396
|
-
{
|
|
2397
|
-
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
2398
|
-
label: "mac address",
|
|
2399
|
-
score: 1
|
|
2400
|
-
},
|
|
2401
|
-
{
|
|
2402
|
-
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
2403
|
-
label: "url",
|
|
2404
|
-
score: .9
|
|
2405
|
-
},
|
|
2406
|
-
{
|
|
2407
|
-
pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
|
|
2408
|
-
label: "address",
|
|
2409
|
-
score: .9
|
|
2410
|
-
},
|
|
2411
|
-
{
|
|
2412
|
-
pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
|
|
2413
|
-
label: "social security number",
|
|
2414
|
-
score: .95,
|
|
2415
|
-
validator: gb.nino
|
|
2416
|
-
},
|
|
2417
|
-
{
|
|
2418
|
-
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
2419
|
-
label: "date",
|
|
2420
|
-
score: .9
|
|
2670
|
+
pattern: `(?<![\\p{L}\\p{N}_.,])(?:${buildPercentWordPattern(AMOUNT_WORDS)}[^\\S\\n]*\\([^\\S\\n]*${PERCENT_TOKEN}[^\\S\\n]*\\)|${PERCENT_RANGE}|${PERCENT_TOKEN})(?![\\p{L}\\p{N}_])`,
|
|
2671
|
+
label: "monetary amount",
|
|
2672
|
+
score: .85
|
|
2421
2673
|
},
|
|
2422
2674
|
...STDNUM_ENTRIES
|
|
2423
2675
|
];
|
|
@@ -2491,6 +2743,41 @@ const DATE_PATTERN_META = Object.freeze({
|
|
|
2491
2743
|
label: "date",
|
|
2492
2744
|
score: 1
|
|
2493
2745
|
});
|
|
2746
|
+
const buildMagnitudePattern = (config) => {
|
|
2747
|
+
const words = [];
|
|
2748
|
+
const caseInsensitiveAbbreviations = [];
|
|
2749
|
+
const caseSensitiveAbbreviations = [];
|
|
2750
|
+
for (const entry of config.magnitudeSuffixes ?? []) {
|
|
2751
|
+
words.push(...entry.words ?? []);
|
|
2752
|
+
caseInsensitiveAbbreviations.push(...entry.abbreviationsCaseInsensitive ?? []);
|
|
2753
|
+
caseSensitiveAbbreviations.push(...entry.abbreviationsCaseSensitive ?? []);
|
|
2754
|
+
}
|
|
2755
|
+
const branches = [];
|
|
2756
|
+
const wordsAlt = toSortedAlternation(words);
|
|
2757
|
+
const abbreviationCiAlt = toSortedAlternation(caseInsensitiveAbbreviations);
|
|
2758
|
+
const abbreviationCsAlt = toSortedAlternation(caseSensitiveAbbreviations);
|
|
2759
|
+
if (wordsAlt) branches.push(`[^\\S\\n\\t]+(?i:(?:${wordsAlt}))\\b`);
|
|
2760
|
+
if (abbreviationCiAlt) branches.push(`[^\\S\\n\\t]?(?i:${abbreviationCiAlt})\\b`);
|
|
2761
|
+
if (abbreviationCsAlt) branches.push(`[^\\S\\n\\t]?(?:${abbreviationCsAlt})\\b`);
|
|
2762
|
+
return branches.length > 0 ? `(?:${branches.join("|")})?` : "";
|
|
2763
|
+
};
|
|
2764
|
+
const buildQuantityFollowerGuard = (config) => {
|
|
2765
|
+
const modifiers = [];
|
|
2766
|
+
const nouns = [];
|
|
2767
|
+
for (const entry of config.shareQuantityTerms ?? []) {
|
|
2768
|
+
modifiers.push(...entry.modifiers ?? []);
|
|
2769
|
+
nouns.push(...entry.nouns);
|
|
2770
|
+
}
|
|
2771
|
+
const modifierAlt = toSortedAlternation(modifiers);
|
|
2772
|
+
const nounAlt = toSortedAlternation(nouns);
|
|
2773
|
+
if (!nounAlt) return "";
|
|
2774
|
+
return `(?![^\\S\\n\\t]+(?i:${modifierAlt ? `(?:(?:${modifierAlt})[^\\S\\n\\t]+){0,3}` : ""}(?:${nounAlt}))\\b)`;
|
|
2775
|
+
};
|
|
2776
|
+
const buildFinancialLexicons = (config) => Object.freeze({
|
|
2777
|
+
magnitude: buildMagnitudePattern(config),
|
|
2778
|
+
quantityFollowerGuard: buildQuantityFollowerGuard(config)
|
|
2779
|
+
});
|
|
2780
|
+
const FINANCIAL_LEXICONS = buildFinancialLexicons(AMOUNT_WORDS);
|
|
2494
2781
|
/**
|
|
2495
2782
|
* Build symbol character class, code alternation,
|
|
2496
2783
|
* and local-name alternation from currencies.json,
|
|
@@ -2530,9 +2817,13 @@ const buildCurrencyPatterns = (data) => {
|
|
|
2530
2817
|
const patterns = [];
|
|
2531
2818
|
const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
|
|
2532
2819
|
const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
|
|
2533
|
-
|
|
2534
|
-
if (
|
|
2535
|
-
if (trailingAlt) patterns.push(`\\b
|
|
2820
|
+
const MAGNITUDE = FINANCIAL_LEXICONS.magnitude;
|
|
2821
|
+
if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${MAGNITUDE}${END}`);
|
|
2822
|
+
if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${MAGNITUDE}${END}`);
|
|
2823
|
+
if (trailingAlt) {
|
|
2824
|
+
const optionalLeadingSymbol = symbols ? `(?<![\\p{L}\\p{N}_])(?:[${symbols}][^\\S\\n\\t]?)?` : "\\b";
|
|
2825
|
+
patterns.push(`${optionalLeadingSymbol}${NUM}${DECIMAL}${MAGNITUDE}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${FINANCIAL_LEXICONS.quantityFollowerGuard}${END}`);
|
|
2826
|
+
}
|
|
2536
2827
|
return patterns;
|
|
2537
2828
|
};
|
|
2538
2829
|
/** Cached promise for currency patterns. Loaded once. */
|
|
@@ -2846,10 +3137,19 @@ const stripQuotes = (value) => {
|
|
|
2846
3137
|
text: stripped
|
|
2847
3138
|
};
|
|
2848
3139
|
};
|
|
2849
|
-
/**
|
|
3140
|
+
/**
|
|
3141
|
+
* Hard stop characters for to-next-comma scanning. A closing
|
|
3142
|
+
* parenthesis or closing bracket terminates a clause just like
|
|
3143
|
+
* an opening one: `State of New York or any other jurisdiction)`
|
|
3144
|
+
* is the tail of a parenthesised insertion, not the start of a
|
|
3145
|
+
* larger phrase that should be absorbed into a jurisdiction span.
|
|
3146
|
+
*/
|
|
2850
3147
|
const COMMA_STOP_CHARS = new Set([
|
|
2851
3148
|
"\n",
|
|
2852
3149
|
"(",
|
|
3150
|
+
")",
|
|
3151
|
+
"[",
|
|
3152
|
+
"]",
|
|
2853
3153
|
" ",
|
|
2854
3154
|
";"
|
|
2855
3155
|
]);
|
|
@@ -3015,6 +3315,27 @@ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STO
|
|
|
3015
3315
|
const warmAddressStopKeywords = async () => {
|
|
3016
3316
|
await loadAddressStopKeywords();
|
|
3017
3317
|
};
|
|
3318
|
+
const MAX_TRIGGER_VALUE_LEN = 100;
|
|
3319
|
+
const MIN_TRIGGER_PHONE_DIGITS = 5;
|
|
3320
|
+
const PHONE_VALUE_START_RE = /^[+(\d]/;
|
|
3321
|
+
const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
|
|
3322
|
+
const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
3323
|
+
const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
3324
|
+
const capAtWordBoundary = (valueText, cap) => {
|
|
3325
|
+
let capped = cap;
|
|
3326
|
+
const isWordChar = (i) => /[\p{L}\p{N}]/u.test(valueText[i] ?? "");
|
|
3327
|
+
while (capped > 0 && isWordChar(capped - 1) && isWordChar(capped)) capped--;
|
|
3328
|
+
return capped;
|
|
3329
|
+
};
|
|
3330
|
+
const isPlausiblePhoneTriggerValue = (value) => {
|
|
3331
|
+
const trimmed = value.trimStart();
|
|
3332
|
+
if (!PHONE_VALUE_START_RE.test(trimmed)) return false;
|
|
3333
|
+
if (ISO_DATE_PREFIX_RE.test(trimmed)) return false;
|
|
3334
|
+
if (INLINE_FIELD_LABEL_RE.test(trimmed)) return false;
|
|
3335
|
+
let digits = 0;
|
|
3336
|
+
for (const ch of trimmed) if (/\d/.test(ch)) digits++;
|
|
3337
|
+
return digits >= MIN_TRIGGER_PHONE_DIGITS;
|
|
3338
|
+
};
|
|
3018
3339
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
3019
3340
|
const remaining = text.slice(triggerEnd);
|
|
3020
3341
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
@@ -3026,21 +3347,11 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3026
3347
|
const stopWords = strategy.stopWords ?? [];
|
|
3027
3348
|
const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
|
|
3028
3349
|
let end = 0;
|
|
3029
|
-
let foundStop = false;
|
|
3030
3350
|
while (end < valueText.length) {
|
|
3031
3351
|
const ch = valueText[end];
|
|
3032
|
-
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch))
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
}
|
|
3036
|
-
if (ch === "." && isSentenceTerminator(valueText, end)) {
|
|
3037
|
-
foundStop = true;
|
|
3038
|
-
break;
|
|
3039
|
-
}
|
|
3040
|
-
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
|
|
3041
|
-
foundStop = true;
|
|
3042
|
-
break;
|
|
3043
|
-
}
|
|
3352
|
+
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) break;
|
|
3353
|
+
if (ch === "." && isSentenceTerminator(valueText, end)) break;
|
|
3354
|
+
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) break;
|
|
3044
3355
|
if (ch === ",") {
|
|
3045
3356
|
const afterComma = valueText.slice(end);
|
|
3046
3357
|
if (DECIMAL_COMMA_RE.test(afterComma)) {
|
|
@@ -3052,12 +3363,12 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3052
3363
|
end += degreeMatch[0].length;
|
|
3053
3364
|
continue;
|
|
3054
3365
|
}
|
|
3055
|
-
foundStop = true;
|
|
3056
3366
|
break;
|
|
3057
3367
|
}
|
|
3058
3368
|
end++;
|
|
3059
3369
|
}
|
|
3060
|
-
|
|
3370
|
+
const lengthCap = strategy.maxLength ?? 100;
|
|
3371
|
+
if (end > lengthCap) end = capAtWordBoundary(valueText, lengthCap);
|
|
3061
3372
|
const rawSlice = valueText.slice(0, end);
|
|
3062
3373
|
const extracted = rawSlice.trim();
|
|
3063
3374
|
if (extracted.length === 0) return null;
|
|
@@ -3071,12 +3382,24 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3071
3382
|
case "to-end-of-line": {
|
|
3072
3383
|
const consumed = remaining.length - valueText.length;
|
|
3073
3384
|
if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
|
|
3074
|
-
const LINE_STOPS = ["\n"];
|
|
3385
|
+
const LINE_STOPS = ["\n", " "];
|
|
3075
3386
|
let end = valueText.length;
|
|
3387
|
+
let foundLineStop = false;
|
|
3076
3388
|
for (const ch of LINE_STOPS) {
|
|
3077
3389
|
const idx = valueText.indexOf(ch);
|
|
3078
|
-
if (idx !== -1 && idx < end)
|
|
3390
|
+
if (idx !== -1 && idx < end) {
|
|
3391
|
+
end = idx;
|
|
3392
|
+
foundLineStop = true;
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
if (label === "phone number") {
|
|
3396
|
+
const inlineLabel = INLINE_FIELD_LABEL_STOP_RE.exec(valueText.slice(0, end));
|
|
3397
|
+
if (inlineLabel) {
|
|
3398
|
+
end = inlineLabel.index;
|
|
3399
|
+
foundLineStop = true;
|
|
3400
|
+
}
|
|
3079
3401
|
}
|
|
3402
|
+
if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
|
|
3080
3403
|
const rawSlice = valueText.slice(0, end);
|
|
3081
3404
|
const extracted = rawSlice.trim();
|
|
3082
3405
|
if (extracted.length === 0) return null;
|
|
@@ -3256,6 +3579,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
3256
3579
|
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
3257
3580
|
if (value) {
|
|
3258
3581
|
if (!applyValidations(value.text, rule.validations)) continue;
|
|
3582
|
+
if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
|
|
3259
3583
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
3260
3584
|
const entityEnd = value.end;
|
|
3261
3585
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
@@ -3597,7 +3921,7 @@ const normalizeHomoglyphs = (text) => {
|
|
|
3597
3921
|
//#endregion
|
|
3598
3922
|
//#region src/filters/false-positives.ts
|
|
3599
3923
|
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
3600
|
-
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
3924
|
+
const POSTAL_CODE_RE$1 = /\d{3}\s?\d{2}/;
|
|
3601
3925
|
const HAS_DIGIT_RE = /\d/;
|
|
3602
3926
|
const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
|
|
3603
3927
|
const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
|
|
@@ -3662,7 +3986,7 @@ const normalizeEntity = (entity) => {
|
|
|
3662
3986
|
text
|
|
3663
3987
|
};
|
|
3664
3988
|
};
|
|
3665
|
-
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
3989
|
+
const EMPTY_GENERIC_ROLES$1 = /* @__PURE__ */ new Set();
|
|
3666
3990
|
/**
|
|
3667
3991
|
* Load generic-roles.json and cache the result on the
|
|
3668
3992
|
* given context. Must be awaited during pipeline init
|
|
@@ -3686,14 +4010,14 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3686
4010
|
return ctx.genericRolesPromise;
|
|
3687
4011
|
};
|
|
3688
4012
|
/** Sync accessor — returns empty set before init. */
|
|
3689
|
-
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
4013
|
+
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES$1;
|
|
3690
4014
|
const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
|
|
3691
4015
|
let _streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3692
4016
|
let _streetTypesPromise = null;
|
|
3693
4017
|
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3694
4018
|
const loadStreetTypeRegex = async () => {
|
|
3695
4019
|
try {
|
|
3696
|
-
const data = (await import("./address-street-types.mjs")).default ?? {};
|
|
4020
|
+
const data = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
|
|
3697
4021
|
const words = /* @__PURE__ */ new Set();
|
|
3698
4022
|
for (const [key, val] of Object.entries(data)) {
|
|
3699
4023
|
if (key.startsWith("_")) continue;
|
|
@@ -3762,7 +4086,7 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3762
4086
|
}
|
|
3763
4087
|
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
|
|
3764
4088
|
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
3765
|
-
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
4089
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE$1.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3766
4090
|
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3767
4091
|
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
|
|
3768
4092
|
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
@@ -3827,6 +4151,23 @@ const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
|
3827
4151
|
/** Sync accessor — returns empty set before init. */
|
|
3828
4152
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
3829
4153
|
/**
|
|
4154
|
+
* Curated dictionary entries that are pure dotted
|
|
4155
|
+
* single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
|
|
4156
|
+
* need targeted suffix guards. The AC search matches
|
|
4157
|
+
* case-insensitively on token boundaries where `.` is not
|
|
4158
|
+
* a word character, so `S.C.` can match inside `U.S.C.`.
|
|
4159
|
+
* Two-segment non-address aliases are too noisy and are
|
|
4160
|
+
* dropped at build time; longer official aliases stay
|
|
4161
|
+
* searchable and are only suppressed when the source text
|
|
4162
|
+
* shows they are the tail of a longer dotted token.
|
|
4163
|
+
* Caller-supplied custom entries are exempted.
|
|
4164
|
+
*/
|
|
4165
|
+
const DOTTED_ACRONYM_RE = /^(?=.{3,}$)\p{L}(?:\.\p{L}){0,3}\.?$/u;
|
|
4166
|
+
const isCuratedNoiseAcronym = (normalized) => DOTTED_ACRONYM_RE.test(normalized);
|
|
4167
|
+
const dottedAcronymSegmentCount = (normalized) => normalized.split(".").filter(Boolean).length;
|
|
4168
|
+
const isShortCuratedNoiseAcronym = (normalized) => isCuratedNoiseAcronym(normalized) && dottedAcronymSegmentCount(normalized) <= 2;
|
|
4169
|
+
const isDottedAcronymSuffixCollision = (fullText, start, matchText) => isCuratedNoiseAcronym(matchText) && /[\p{L}]\.$/u.test(fullText.slice(Math.max(0, start - 2), start));
|
|
4170
|
+
/**
|
|
3830
4171
|
* Common EU given names present in the stopwords-iso dataset
|
|
3831
4172
|
* but absent from the first-name corpus. Without this
|
|
3832
4173
|
* supplementary set, these names would pass through the
|
|
@@ -3962,7 +4303,7 @@ let streetTypeReLoaded = false;
|
|
|
3962
4303
|
const loadStreetTypeRe = async () => {
|
|
3963
4304
|
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
3964
4305
|
try {
|
|
3965
|
-
const config = (await import("./address-street-types.mjs")).default ?? {};
|
|
4306
|
+
const config = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
|
|
3966
4307
|
const words = [];
|
|
3967
4308
|
for (const value of Object.values(config)) {
|
|
3968
4309
|
if (!Array.isArray(value)) continue;
|
|
@@ -4076,6 +4417,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4076
4417
|
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
4077
4418
|
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
4078
4419
|
if (normalized.length === 0) return;
|
|
4420
|
+
if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
|
|
4079
4421
|
const lower = normalized.toLowerCase();
|
|
4080
4422
|
const existing = patternIndex.get(lower);
|
|
4081
4423
|
if (existing !== void 0) {
|
|
@@ -4153,6 +4495,7 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
4153
4495
|
const addNameEntry = (name, source) => {
|
|
4154
4496
|
const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
|
|
4155
4497
|
if (normalized.length === 0) return;
|
|
4498
|
+
if (isCuratedNoiseAcronym(normalized)) return;
|
|
4156
4499
|
const lower = normalized.toLowerCase();
|
|
4157
4500
|
const existing = patternIndex.get(lower);
|
|
4158
4501
|
if (existing !== void 0) {
|
|
@@ -4239,12 +4582,14 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
4239
4582
|
const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
|
|
4240
4583
|
const customLabels = customEdgesAreValid ? customPatternLabels : [];
|
|
4241
4584
|
if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
|
|
4242
|
-
const
|
|
4243
|
-
|
|
4585
|
+
const acronymMatchesAcronym = !(pattern.length > 0 && pattern.length <= 5 && ALL_UPPER_RE.test(pattern)) || ALL_UPPER_RE.test(matchText);
|
|
4586
|
+
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && acronymMatchesAcronym && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
|
|
4587
|
+
const filteredCuratedLabels = isDottedAcronymSuffixCollision(fullText, match.start, matchText) ? [] : curatedLabels;
|
|
4588
|
+
if (filteredCuratedLabels.length === 0 && customLabels.length === 0) continue;
|
|
4244
4589
|
const entry = {
|
|
4245
4590
|
start: match.start,
|
|
4246
4591
|
end: match.end,
|
|
4247
|
-
labels:
|
|
4592
|
+
labels: filteredCuratedLabels,
|
|
4248
4593
|
customLabels,
|
|
4249
4594
|
sources,
|
|
4250
4595
|
text: matchText,
|
|
@@ -4310,6 +4655,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
4310
4655
|
const first = chain.at(0);
|
|
4311
4656
|
const last = chain.at(-1);
|
|
4312
4657
|
if (!first || !last) continue;
|
|
4658
|
+
if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
|
|
4313
4659
|
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
4314
4660
|
const score = chain.length >= 2 ? .9 : .5;
|
|
4315
4661
|
if (chain.length === 1) {
|
|
@@ -4405,6 +4751,95 @@ const extendCityDistricts = (entities, fullText) => {
|
|
|
4405
4751
|
* by a capitalized word (for "Miroslav Braňka" when
|
|
4406
4752
|
* only "Braňka" matched).
|
|
4407
4753
|
*/
|
|
4754
|
+
/**
|
|
4755
|
+
* Defined-term marker: an opening typographic or straight
|
|
4756
|
+
* quote enclosing the chain start, AND a closing quote
|
|
4757
|
+
* within a short window followed by a
|
|
4758
|
+
* definitional cue (`means`, `shall mean`, `shall have
|
|
4759
|
+
* the meaning(s)`, `refers to`). Legal documents reserve
|
|
4760
|
+
* this construction for defined terms; the contents are
|
|
4761
|
+
* not personal names even when individual tokens collide
|
|
4762
|
+
* with the name corpus.
|
|
4763
|
+
*
|
|
4764
|
+
* Plain quotations like `"John Unknown" said ...` do NOT
|
|
4765
|
+
* count: there is no definitional cue, so the trailing
|
|
4766
|
+
* surname extension is still allowed to absorb `Unknown`.
|
|
4767
|
+
*/
|
|
4768
|
+
const OPENING_QUOTES = new Set([
|
|
4769
|
+
"\"",
|
|
4770
|
+
"'",
|
|
4771
|
+
"“",
|
|
4772
|
+
"„",
|
|
4773
|
+
"‟",
|
|
4774
|
+
"‘",
|
|
4775
|
+
"‛",
|
|
4776
|
+
"«"
|
|
4777
|
+
]);
|
|
4778
|
+
const CLOSING_QUOTES = new Set([
|
|
4779
|
+
"\"",
|
|
4780
|
+
"'",
|
|
4781
|
+
"”",
|
|
4782
|
+
"’",
|
|
4783
|
+
"»",
|
|
4784
|
+
"“"
|
|
4785
|
+
]);
|
|
4786
|
+
const DEFINED_TERM_CUE_RE = /^[\s,]*(?:means?|shall\s+means?|shall\s+have\s+the\s+meanings?|refers?\s+to|has\s+the\s+meanings?|is\s+defined)\b/iu;
|
|
4787
|
+
const DEFINED_TERM_LOOKAHEAD = 120;
|
|
4788
|
+
const DEFINED_TERM_LOOKBEHIND = 80;
|
|
4789
|
+
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
4790
|
+
const isLetter = (ch) => ch !== void 0 && /^\p{L}$/u.test(ch);
|
|
4791
|
+
const isApostropheInsideWord = (text, index) => isLetter(text[index - 1]) && isLetter(text[index + 1]);
|
|
4792
|
+
const isQuoteBoundary = (text, index) => {
|
|
4793
|
+
const ch = text[index];
|
|
4794
|
+
if (ch !== "'" && ch !== "’") return true;
|
|
4795
|
+
return !isApostropheInsideWord(text, index);
|
|
4796
|
+
};
|
|
4797
|
+
const findDefinedTermQuoteContent = (text, start) => {
|
|
4798
|
+
const min = Math.max(0, start - DEFINED_TERM_LOOKBEHIND);
|
|
4799
|
+
let quoteStart = -1;
|
|
4800
|
+
for (let i = start - 1; i >= min; i--) {
|
|
4801
|
+
const ch = text[i];
|
|
4802
|
+
if (ch === "\n") break;
|
|
4803
|
+
if (ch && OPENING_QUOTES.has(ch) && isQuoteBoundary(text, i)) {
|
|
4804
|
+
quoteStart = i;
|
|
4805
|
+
break;
|
|
4806
|
+
}
|
|
4807
|
+
if (ch && CLOSING_QUOTES.has(ch) && isQuoteBoundary(text, i)) break;
|
|
4808
|
+
}
|
|
4809
|
+
if (quoteStart === -1) return null;
|
|
4810
|
+
const max = Math.min(text.length, quoteStart + 1 + DEFINED_TERM_LOOKAHEAD);
|
|
4811
|
+
for (let i = start; i < max; i++) {
|
|
4812
|
+
const ch = text[i];
|
|
4813
|
+
if (!ch || !CLOSING_QUOTES.has(ch) || !isQuoteBoundary(text, i)) continue;
|
|
4814
|
+
const after = text.slice(i + 1, max);
|
|
4815
|
+
if (!DEFINED_TERM_CUE_RE.test(after)) return null;
|
|
4816
|
+
return {
|
|
4817
|
+
content: text.slice(quoteStart + 1, i),
|
|
4818
|
+
afterClosingQuote: after
|
|
4819
|
+
};
|
|
4820
|
+
}
|
|
4821
|
+
return null;
|
|
4822
|
+
};
|
|
4823
|
+
const FIRST_WORD_RE = /^\p{L}+/u;
|
|
4824
|
+
const WORD_RE = /\p{L}+/gu;
|
|
4825
|
+
const startsWithKnownFirstName = (quoteContent, ctx) => {
|
|
4826
|
+
const firstWord = FIRST_WORD_RE.exec(quoteContent.trim())?.[0];
|
|
4827
|
+
if (!firstWord) return false;
|
|
4828
|
+
return new Set(getNameCorpusFirstNames(ctx).map((name) => name.toLowerCase())).has(firstWord.toLowerCase());
|
|
4829
|
+
};
|
|
4830
|
+
const hasPersonRoleDefinition = (afterClosingQuote, ctx) => {
|
|
4831
|
+
const roleWords = afterClosingQuote.replace(DEFINED_TERM_CUE_RE, "").match(WORD_RE)?.slice(0, 8) ?? [];
|
|
4832
|
+
if (roleWords.length === 0) return false;
|
|
4833
|
+
const genericRoles = ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
4834
|
+
return roleWords.some((word) => genericRoles.has(word.toLowerCase()));
|
|
4835
|
+
};
|
|
4836
|
+
const isSuppressibleDefinedTermQuote = (text, start, ctx) => {
|
|
4837
|
+
const definedTermQuote = findDefinedTermQuoteContent(text, start);
|
|
4838
|
+
if (definedTermQuote === null) return false;
|
|
4839
|
+
const words = definedTermQuote.content.match(WORD_RE) ?? [];
|
|
4840
|
+
if (words.length >= 2 && startsWithKnownFirstName(definedTermQuote.content, ctx) && hasPersonRoleDefinition(definedTermQuote.afterClosingQuote, ctx)) return false;
|
|
4841
|
+
return words.length >= 2;
|
|
4842
|
+
};
|
|
4408
4843
|
const extendPersonName = (text, start, end, ctx) => {
|
|
4409
4844
|
let newEnd = end;
|
|
4410
4845
|
let pos = newEnd;
|
|
@@ -4415,7 +4850,7 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
4415
4850
|
if (!UPPER_START_RE.test(char)) break;
|
|
4416
4851
|
let wordEnd = wordStart;
|
|
4417
4852
|
while (wordEnd < text.length && !/\s/.test(text[wordEnd] ?? "")) wordEnd++;
|
|
4418
|
-
const stripped = text.slice(wordStart, wordEnd).replace(/[
|
|
4853
|
+
const stripped = text.slice(wordStart, wordEnd).replace(/[,;.”"’'“»]+$/, "");
|
|
4419
4854
|
if (stripped.length < 2) break;
|
|
4420
4855
|
const lower = stripped.toLowerCase();
|
|
4421
4856
|
if (getStopwords(ctx).has(lower) || getPersonStopwords(ctx).has(lower)) break;
|
|
@@ -4438,6 +4873,15 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
4438
4873
|
* cluster occasionally absorbs.
|
|
4439
4874
|
*/
|
|
4440
4875
|
const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
|
|
4876
|
+
const POSTAL_ADJACENT = `\\p{L}\\p{N}_${DASH_INNER}`;
|
|
4877
|
+
const POSTAL_CODE_RE = new RegExp(`(?<![${POSTAL_ADJACENT}])(?:\\d{3}\\s\\d{2}|\\d{2}${DASH}\\d{3}|\\d{5}${DASH}\\d{3}|\\d{5}${DASH}\\d{4})(?![${POSTAL_ADJACENT}])`, "gu");
|
|
4878
|
+
const BR_CEP_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{3}$`, "u");
|
|
4879
|
+
const US_ZIP_PLUS_FOUR_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{4}$`, "u");
|
|
4880
|
+
const US_STATE_ABBREV_BEFORE_ZIP_RE = new RegExp(`(?:^|[^A-Za-z0-9])(A[KLRZ]|C[AOT]|D[CE]|F[LM]|G[AU]|HI|I[ADLN]|K[SY]|LA|M[ADEHINOPST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\\s*,?\\s*$`, "u");
|
|
4881
|
+
const US_ZIP_CONTEXT_WINDOW = 120;
|
|
4882
|
+
const US_CITY_ZIP_GAP_RE = /^[\s,]+$/u;
|
|
4883
|
+
const HOUSE_NUMBER_BEFORE_STREET_RE = /\b\d{1,6}(?:[-/]\d{1,6})?\s+(?:\p{Lu}\p{L}+[^\S\n\t]+){0,4}$/u;
|
|
4884
|
+
const HOUSE_NUMBER_AFTER_STREET_RE = /^[^\S\n\t]+\d{1,6}(?:[-/]\d{1,6})?\b/u;
|
|
4441
4885
|
let cachedBoundaryRe = null;
|
|
4442
4886
|
const loadBoundaryWords = async () => {
|
|
4443
4887
|
try {
|
|
@@ -4452,7 +4896,7 @@ let cachedBrCepContextPromise = null;
|
|
|
4452
4896
|
const loadBrCueWords = async () => {
|
|
4453
4897
|
const sources = await Promise.all([(async () => {
|
|
4454
4898
|
try {
|
|
4455
|
-
return (await import("./address-street-types.mjs")).default["pt-br"];
|
|
4899
|
+
return (await import("./address-street-types.mjs").then((n) => n.n)).default["pt-br"];
|
|
4456
4900
|
} catch {
|
|
4457
4901
|
return;
|
|
4458
4902
|
}
|
|
@@ -4496,6 +4940,48 @@ const hasBrCueNearby = (fullText, start, end, re) => {
|
|
|
4496
4940
|
const window = fullText.slice(windowStart, windowEnd);
|
|
4497
4941
|
return new RegExp(re.source, re.flags.replace("g", "")).test(window);
|
|
4498
4942
|
};
|
|
4943
|
+
const getUsStateSeedBeforeZip = (fullText, start) => {
|
|
4944
|
+
const stateWindowStart = Math.max(0, start - 24);
|
|
4945
|
+
const stateWindow = fullText.slice(stateWindowStart, start);
|
|
4946
|
+
const match = US_STATE_ABBREV_BEFORE_ZIP_RE.exec(stateWindow);
|
|
4947
|
+
const state = match?.[1];
|
|
4948
|
+
if (!match || !state) return null;
|
|
4949
|
+
const stateOffset = match[0].indexOf(state);
|
|
4950
|
+
const stateStart = stateWindowStart + match.index + stateOffset;
|
|
4951
|
+
return {
|
|
4952
|
+
type: "state",
|
|
4953
|
+
start: stateStart,
|
|
4954
|
+
end: stateStart + state.length,
|
|
4955
|
+
text: state
|
|
4956
|
+
};
|
|
4957
|
+
};
|
|
4958
|
+
const hasHouseNumberNearStreetWord = (fullText, seed) => {
|
|
4959
|
+
if (/\d/.test(seed.text)) return true;
|
|
4960
|
+
const before = fullText.slice(Math.max(0, seed.start - 50), seed.start);
|
|
4961
|
+
if (HOUSE_NUMBER_BEFORE_STREET_RE.test(before)) return true;
|
|
4962
|
+
const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
|
|
4963
|
+
return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
|
|
4964
|
+
};
|
|
4965
|
+
const getUsZipPlusFourContext = (fullText, start, seeds) => {
|
|
4966
|
+
const stateSeed = getUsStateSeedBeforeZip(fullText, start);
|
|
4967
|
+
if (stateSeed !== null) return {
|
|
4968
|
+
stateSeed,
|
|
4969
|
+
hasContext: true
|
|
4970
|
+
};
|
|
4971
|
+
return {
|
|
4972
|
+
stateSeed: null,
|
|
4973
|
+
hasContext: seeds.some((seed) => {
|
|
4974
|
+
if (Math.abs(seed.start - start) > US_ZIP_CONTEXT_WINDOW) return false;
|
|
4975
|
+
if (seed.type === "address-trigger") return true;
|
|
4976
|
+
if (seed.type === "city" && seed.end <= start) {
|
|
4977
|
+
const gap = fullText.slice(seed.end, start);
|
|
4978
|
+
return US_CITY_ZIP_GAP_RE.test(gap);
|
|
4979
|
+
}
|
|
4980
|
+
if (seed.type === "street-word") return hasHouseNumberNearStreetWord(fullText, seed);
|
|
4981
|
+
return false;
|
|
4982
|
+
})
|
|
4983
|
+
};
|
|
4984
|
+
};
|
|
4499
4985
|
/**
|
|
4500
4986
|
* Build regex for boundary words. Matches any
|
|
4501
4987
|
* boundary word preceded by a word boundary.
|
|
@@ -4520,7 +5006,7 @@ const getBoundaryRe = async () => {
|
|
|
4520
5006
|
const buildStreetTypePatterns = async () => {
|
|
4521
5007
|
let config = {};
|
|
4522
5008
|
try {
|
|
4523
|
-
config = (await import("./address-street-types.mjs")).default;
|
|
5009
|
+
config = (await import("./address-street-types.mjs").then((n) => n.n)).default;
|
|
4524
5010
|
} catch {
|
|
4525
5011
|
return [];
|
|
4526
5012
|
}
|
|
@@ -4565,13 +5051,22 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
4565
5051
|
text: e.text
|
|
4566
5052
|
});
|
|
4567
5053
|
}
|
|
4568
|
-
const postalRe =
|
|
5054
|
+
const postalRe = POSTAL_CODE_RE;
|
|
5055
|
+
postalRe.lastIndex = 0;
|
|
4569
5056
|
let postalMatch;
|
|
4570
5057
|
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
4571
5058
|
const start = postalMatch.index;
|
|
4572
5059
|
const end = start + postalMatch[0].length;
|
|
4573
5060
|
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4574
|
-
if (
|
|
5061
|
+
if (BR_CEP_SHAPE_RE.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
|
|
5062
|
+
if (US_ZIP_PLUS_FOUR_SHAPE_RE.test(postalMatch[0])) {
|
|
5063
|
+
const usContext = getUsZipPlusFourContext(fullText, start, seeds);
|
|
5064
|
+
if (!usContext.hasContext) continue;
|
|
5065
|
+
const stateSeed = usContext.stateSeed;
|
|
5066
|
+
if (stateSeed !== null) {
|
|
5067
|
+
if (!seeds.some((seed) => seed.start === stateSeed.start && seed.end === stateSeed.end)) seeds.push(stateSeed);
|
|
5068
|
+
}
|
|
5069
|
+
}
|
|
4575
5070
|
seeds.push({
|
|
4576
5071
|
type: "postal-code",
|
|
4577
5072
|
start,
|
|
@@ -4650,6 +5145,7 @@ const scoreCluster = (cluster) => {
|
|
|
4650
5145
|
let score = .5;
|
|
4651
5146
|
if (types.has("postal-code")) score += .15;
|
|
4652
5147
|
if (types.has("city")) score += .15;
|
|
5148
|
+
if (types.has("state")) score += .15;
|
|
4653
5149
|
if (types.has("street-word")) score += .15;
|
|
4654
5150
|
if (types.has("address-trigger")) score += .1;
|
|
4655
5151
|
return Math.min(score, .95);
|
|
@@ -4836,7 +5332,44 @@ const BARE_STOPWORDS = new Set([
|
|
|
4836
5332
|
"Splatnost",
|
|
4837
5333
|
"Variabilní",
|
|
4838
5334
|
"Konstantní",
|
|
4839
|
-
"Specifický"
|
|
5335
|
+
"Specifický",
|
|
5336
|
+
"Section",
|
|
5337
|
+
"Sections",
|
|
5338
|
+
"Article",
|
|
5339
|
+
"Articles",
|
|
5340
|
+
"Schedule",
|
|
5341
|
+
"Schedules",
|
|
5342
|
+
"Exhibit",
|
|
5343
|
+
"Exhibits",
|
|
5344
|
+
"Annex",
|
|
5345
|
+
"Annexes",
|
|
5346
|
+
"Appendix",
|
|
5347
|
+
"Appendices",
|
|
5348
|
+
"Clause",
|
|
5349
|
+
"Clauses",
|
|
5350
|
+
"Chapter",
|
|
5351
|
+
"Chapters",
|
|
5352
|
+
"Paragraph",
|
|
5353
|
+
"Paragraphs",
|
|
5354
|
+
"Subsection",
|
|
5355
|
+
"Subsections",
|
|
5356
|
+
"Form",
|
|
5357
|
+
"Page",
|
|
5358
|
+
"Pages",
|
|
5359
|
+
"Item",
|
|
5360
|
+
"Items",
|
|
5361
|
+
"Note",
|
|
5362
|
+
"Notes",
|
|
5363
|
+
"Rule",
|
|
5364
|
+
"Rules",
|
|
5365
|
+
"Attachment",
|
|
5366
|
+
"Attachments",
|
|
5367
|
+
"Volume",
|
|
5368
|
+
"Volumes",
|
|
5369
|
+
"Book",
|
|
5370
|
+
"Books",
|
|
5371
|
+
"Part",
|
|
5372
|
+
"Parts"
|
|
4840
5373
|
]);
|
|
4841
5374
|
const NEAR_MISS_BAND = .15;
|
|
4842
5375
|
const BOOST_PER_NEIGHBOUR = .05;
|
|
@@ -4909,21 +5442,23 @@ const initPrepositions = () => {
|
|
|
4909
5442
|
};
|
|
4910
5443
|
const getAddressPreps = () => _addressPreps ?? /* @__PURE__ */ new Set();
|
|
4911
5444
|
const getTemporalPreps = () => _temporalPreps ?? /* @__PURE__ */ new Set();
|
|
4912
|
-
|
|
5445
|
+
const buildStreetAbbrevs = (data) => {
|
|
5446
|
+
const abbrevs = /* @__PURE__ */ new Set();
|
|
5447
|
+
for (const [key, words] of Object.entries(data)) {
|
|
5448
|
+
if (key.startsWith("_")) continue;
|
|
5449
|
+
if (!Array.isArray(words)) continue;
|
|
5450
|
+
for (const word of words) if (typeof word === "string" && word.includes(".")) abbrevs.add(word.toLowerCase());
|
|
5451
|
+
}
|
|
5452
|
+
return abbrevs;
|
|
5453
|
+
};
|
|
5454
|
+
let _streetAbbrevs = buildStreetAbbrevs(address_street_types_default);
|
|
4913
5455
|
let _streetAbbrevsPromise = null;
|
|
4914
5456
|
const loadStreetAbbrevs = async () => {
|
|
4915
5457
|
try {
|
|
4916
|
-
const mod = await import("./address-street-types.mjs");
|
|
4917
|
-
|
|
4918
|
-
const abbrevs = /* @__PURE__ */ new Set();
|
|
4919
|
-
for (const [key, words] of Object.entries(data)) {
|
|
4920
|
-
if (key.startsWith("_")) continue;
|
|
4921
|
-
if (!Array.isArray(words)) continue;
|
|
4922
|
-
for (const w of words) if (w.includes(".")) abbrevs.add(w.toLowerCase());
|
|
4923
|
-
}
|
|
4924
|
-
_streetAbbrevs = abbrevs;
|
|
5458
|
+
const mod = await import("./address-street-types.mjs").then((n) => n.n);
|
|
5459
|
+
_streetAbbrevs = buildStreetAbbrevs(mod.default ?? mod);
|
|
4925
5460
|
} catch {
|
|
4926
|
-
_streetAbbrevs
|
|
5461
|
+
_streetAbbrevs ??= /* @__PURE__ */ new Set();
|
|
4927
5462
|
}
|
|
4928
5463
|
};
|
|
4929
5464
|
/** Ensure street abbreviation data is loaded. */
|
|
@@ -5035,7 +5570,7 @@ const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
|
5035
5570
|
}
|
|
5036
5571
|
return results;
|
|
5037
5572
|
};
|
|
5038
|
-
const ORPHAN_STREET_RE =
|
|
5573
|
+
const ORPHAN_STREET_RE = /^[^\S\n]*(\p{Lu}[\p{Ll}\p{Lu}]+(?:[^\S\n]+[\p{Lu}\p{Ll}][\p{Ll}]+)*[^\S\n]+\d{2,4}[a-zA-Z]?)[^\S\n]*$/gmu;
|
|
5039
5574
|
/**
|
|
5040
5575
|
* In the header zone (top 15%), find standalone lines
|
|
5041
5576
|
* matching "[Uppercase word(s)] [number]" that sit
|
|
@@ -5413,6 +5948,7 @@ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-lis
|
|
|
5413
5948
|
* of `\s` to avoid merging entities across newlines.
|
|
5414
5949
|
*/
|
|
5415
5950
|
const GAP_PATTERN = /^[ \t,\-]+$/;
|
|
5951
|
+
const isLegalFormOrganization = (entity) => entity.label === "organization" && entity.source === DETECTION_SOURCES.LEGAL_FORM;
|
|
5416
5952
|
/**
|
|
5417
5953
|
* Build a set of word boundary offsets for the full
|
|
5418
5954
|
* text using `Intl.Segmenter`. Returns a sorted array
|
|
@@ -5552,7 +6088,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
5552
6088
|
break;
|
|
5553
6089
|
}
|
|
5554
6090
|
}
|
|
5555
|
-
if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
6091
|
+
if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev) && isLegalFormOrganization(entity) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
5556
6092
|
prev.end = entity.end;
|
|
5557
6093
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
5558
6094
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -5952,6 +6488,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
5952
6488
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
5953
6489
|
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
5954
6490
|
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
6491
|
+
const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
|
|
6492
|
+
const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
|
|
5955
6493
|
const shouldReplace = (a, b) => {
|
|
5956
6494
|
const aLen = a.end - a.start;
|
|
5957
6495
|
const bLen = b.end - b.start;
|
|
@@ -5967,6 +6505,31 @@ const shouldReplace = (a, b) => {
|
|
|
5967
6505
|
/** Labels where colons are structurally significant. */
|
|
5968
6506
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
5969
6507
|
/**
|
|
6508
|
+
* Labels whose entities should have a trailing sentence
|
|
6509
|
+
* `.` stripped during sanitisation. Restricted to
|
|
6510
|
+
* proper-noun-style labels where a final period is
|
|
6511
|
+
* almost always the sentence terminator that ran into
|
|
6512
|
+
* the capture, not a structural part of the value.
|
|
6513
|
+
* Numeric labels (`date`, `date of birth`, `phone
|
|
6514
|
+
* number`, `monetary amount`, `time`) and `person`
|
|
6515
|
+
* stay out — German writes `21. März`, post-nominal
|
|
6516
|
+
* degrees write `M.Sc.`, times write `5:00 p.m.`, and
|
|
6517
|
+
* stripping the dot would corrupt those spans.
|
|
6518
|
+
*/
|
|
6519
|
+
const PERIOD_STRIPPED_LABELS = new Set([
|
|
6520
|
+
"organization",
|
|
6521
|
+
"location",
|
|
6522
|
+
"address"
|
|
6523
|
+
]);
|
|
6524
|
+
const ADDRESS_FINAL_TOKEN_RE = /(?:^|[\s,])([\p{L}\p{M}.]+\.)$/u;
|
|
6525
|
+
const LOCATION_FINAL_DOTTED_ABBREV_RE = /(?:^|[\s,])(?:\p{Lu}\.){2,}$/u;
|
|
6526
|
+
const hasKnownAddressFinalAbbrev = (text) => {
|
|
6527
|
+
const finalToken = ADDRESS_FINAL_TOKEN_RE.exec(text)?.[1];
|
|
6528
|
+
if (!finalToken) return false;
|
|
6529
|
+
return getStreetAbbrevs().has(finalToken.toLowerCase());
|
|
6530
|
+
};
|
|
6531
|
+
const hasLocationFinalAbbrev = (text) => LOCATION_FINAL_DOTTED_ABBREV_RE.test(text);
|
|
6532
|
+
/**
|
|
5970
6533
|
* Labels whose detectors emit precise, evidence-backed spans. When
|
|
5971
6534
|
* one of these fires at the exact same offsets as a fuzzier
|
|
5972
6535
|
* `address` hit (city dictionary lookup, address-seed cluster), the
|
|
@@ -6039,16 +6602,63 @@ const resolveSameSpanLabelConflicts = (entities) => {
|
|
|
6039
6602
|
if (dropped.size === 0) return entities;
|
|
6040
6603
|
return entities.filter((e) => !dropped.has(e));
|
|
6041
6604
|
};
|
|
6605
|
+
/**
|
|
6606
|
+
* Trailing typographic punctuation that detectors
|
|
6607
|
+
* occasionally swallow when a capture runs to the end
|
|
6608
|
+
* of a sentence or quoted phrase. Stripped from every
|
|
6609
|
+
* non-literal, non-locked entity. Curated dictionary and
|
|
6610
|
+
* gazetteer entries with punctuation that is clearly part of
|
|
6611
|
+
* the literal (`Hello bank!`, `"Juez y parte"`) keep their
|
|
6612
|
+
* own boundaries. Generated/extended spans from the same
|
|
6613
|
+
* sources still pass through cleanup so dangling punctuation
|
|
6614
|
+
* does not become part of the redaction
|
|
6615
|
+
* (e.g. `Bond Hedge Documentation"` →
|
|
6616
|
+
* `Bond Hedge Documentation`).
|
|
6617
|
+
*
|
|
6618
|
+
* `)` is deliberately omitted — monetary amounts are
|
|
6619
|
+
* extended to include trailing "(slovy ...)" / "(in
|
|
6620
|
+
* words ...)" parentheticals where the closing paren
|
|
6621
|
+
* is structural, and stripping it would leave the open
|
|
6622
|
+
* paren dangling. `.` is also omitted because the
|
|
6623
|
+
* trailing-period rule below has label-aware handling
|
|
6624
|
+
* (legal-form abbreviations keep their dot).
|
|
6625
|
+
*/
|
|
6626
|
+
const TRAILING_PUNCT_CLASS = `["“”‘’'»!?]`;
|
|
6627
|
+
/**
|
|
6628
|
+
* Leading typographic punctuation that detectors
|
|
6629
|
+
* occasionally swallow when a capture starts at an
|
|
6630
|
+
* opening quote. `(` is deliberately omitted — it is
|
|
6631
|
+
* almost always the opening of a structural
|
|
6632
|
+
* parenthetical (registration number group, monetary
|
|
6633
|
+
* "(slovy ...)" extension) that the detector
|
|
6634
|
+
* intentionally captured.
|
|
6635
|
+
*/
|
|
6636
|
+
const LEADING_PUNCT_CLASS = `["“”‘’'«¿¡]`;
|
|
6637
|
+
const STRIP_BY_LABEL = {
|
|
6638
|
+
colon: /[\s,;]+/,
|
|
6639
|
+
default: /[\s:,;]+/
|
|
6640
|
+
};
|
|
6641
|
+
const LEADING_TRIM_BY_LABEL = {
|
|
6642
|
+
colon: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.colon.source}|${LEADING_PUNCT_CLASS})+`),
|
|
6643
|
+
default: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.default.source}|${LEADING_PUNCT_CLASS})+`)
|
|
6644
|
+
};
|
|
6645
|
+
const TRAILING_TRIM_BY_LABEL = {
|
|
6646
|
+
colon: new RegExp(`(?:${STRIP_BY_LABEL.colon.source}|${TRAILING_PUNCT_CLASS})+$`),
|
|
6647
|
+
default: new RegExp(`(?:${STRIP_BY_LABEL.default.source}|${TRAILING_PUNCT_CLASS})+$`)
|
|
6648
|
+
};
|
|
6042
6649
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
6043
6650
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
6044
|
-
if (hasLockedBoundary(e)) return [e];
|
|
6045
|
-
const
|
|
6046
|
-
const
|
|
6651
|
+
if (hasLockedBoundary(e) || hasCuratedLiteralBoundary(e)) return [e];
|
|
6652
|
+
const stripKind = COLON_LABELS.has(e.label) ? "colon" : "default";
|
|
6653
|
+
const leadRe = LEADING_TRIM_BY_LABEL[stripKind];
|
|
6654
|
+
const trailRe = TRAILING_TRIM_BY_LABEL[stripKind];
|
|
6655
|
+
const leadTrimmed = e.text.replace(leadRe, "");
|
|
6047
6656
|
const lead = e.text.length - leadTrimmed.length;
|
|
6048
|
-
let cleaned = leadTrimmed.replace(
|
|
6049
|
-
if (e.label
|
|
6050
|
-
if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6657
|
+
let cleaned = leadTrimmed.replace(trailRe, "");
|
|
6658
|
+
if (PERIOD_STRIPPED_LABELS.has(e.label) && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
6659
|
+
if (!(getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix)) || e.label === "address" && hasKnownAddressFinalAbbrev(cleaned) || e.label === "location" && hasLocationFinalAbbrev(cleaned))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6051
6660
|
}
|
|
6661
|
+
cleaned = cleaned.replace(trailRe, "");
|
|
6052
6662
|
if (cleaned.length === 0) return [];
|
|
6053
6663
|
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
6054
6664
|
const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
|
|
@@ -6105,7 +6715,7 @@ let amountWordsLoaded = false;
|
|
|
6105
6715
|
const getAmountWordsRe = async () => {
|
|
6106
6716
|
if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
|
|
6107
6717
|
try {
|
|
6108
|
-
const alt = (await import("./amount-words.mjs")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
6718
|
+
const alt = (await import("./amount-words.mjs").then((n) => n.n)).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
6109
6719
|
amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
|
|
6110
6720
|
} catch {
|
|
6111
6721
|
amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
|