@stll/anonymize-wasm 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-street-types.mjs +20 -1
- package/dist/allow-list.mjs +12 -0
- package/dist/amount-words.mjs +472 -36
- package/dist/constants.d.mts +2 -0
- package/dist/constants.mjs +85 -0
- package/dist/constants.mjs.map +1 -0
- package/dist/constants2.d.mts +65 -0
- package/dist/generic-roles.mjs +1 -0
- package/dist/rolldown-runtime.mjs +13 -0
- package/dist/structural-single-cap-prefixes.mjs +99 -0
- package/dist/structural-single-cap-prefixes.mjs.map +1 -0
- package/dist/triggers.en.mjs +48 -1
- package/dist/wasm.d.mts +12 -38
- package/dist/wasm.mjs +722 -156
- package/dist/wasm.mjs.map +1 -1
- package/package.json +6 -1
package/dist/wasm.mjs
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
|
|
2
|
+
import { t as amount_words_default } from "./amount-words.mjs";
|
|
3
|
+
import { t as address_street_types_default } from "./address-street-types.mjs";
|
|
1
4
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
2
5
|
import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
3
6
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
@@ -12,70 +15,6 @@ const getTextSearch = () => {
|
|
|
12
15
|
};
|
|
13
16
|
//#endregion
|
|
14
17
|
//#region src/types.ts
|
|
15
|
-
/**
|
|
16
|
-
* Source of a detected entity span.
|
|
17
|
-
* Ordered by detection layer in the pipeline.
|
|
18
|
-
*/
|
|
19
|
-
const DETECTION_SOURCES = {
|
|
20
|
-
TRIGGER: "trigger",
|
|
21
|
-
REGEX: "regex",
|
|
22
|
-
DENY_LIST: "deny-list",
|
|
23
|
-
LEGAL_FORM: "legal-form",
|
|
24
|
-
GAZETTEER: "gazetteer",
|
|
25
|
-
NER: "ner",
|
|
26
|
-
COREFERENCE: "coreference"
|
|
27
|
-
};
|
|
28
|
-
/**
|
|
29
|
-
* Priority levels for detection sources.
|
|
30
|
-
* Higher = more structurally reliable. Used during
|
|
31
|
-
* overlap resolution so deterministic detectors beat
|
|
32
|
-
* probabilistic ones regardless of raw score.
|
|
33
|
-
*/
|
|
34
|
-
const DETECTOR_PRIORITY = {
|
|
35
|
-
gazetteer: 5,
|
|
36
|
-
trigger: 4,
|
|
37
|
-
"legal-form": 3,
|
|
38
|
-
regex: 3,
|
|
39
|
-
"deny-list": 2,
|
|
40
|
-
coreference: 2,
|
|
41
|
-
ner: 1
|
|
42
|
-
};
|
|
43
|
-
/**
|
|
44
|
-
* Anonymisation operator types. Each operator defines
|
|
45
|
-
* how a confirmed entity is replaced in the output.
|
|
46
|
-
*/
|
|
47
|
-
const OPERATOR_TYPES = ["replace", "redact"];
|
|
48
|
-
/**
|
|
49
|
-
* Canonical entity labels used across the pipeline.
|
|
50
|
-
* NER models may use different native labels; the bench
|
|
51
|
-
* NER wrapper maps model output to these canonical names.
|
|
52
|
-
*
|
|
53
|
-
* These labels are ephemeral: entities are regenerated on
|
|
54
|
-
* every pipeline run and never persisted to the database.
|
|
55
|
-
* Renaming a label here requires no migration.
|
|
56
|
-
*/
|
|
57
|
-
const DEFAULT_ENTITY_LABELS = [
|
|
58
|
-
"person",
|
|
59
|
-
"organization",
|
|
60
|
-
"phone number",
|
|
61
|
-
"address",
|
|
62
|
-
"email address",
|
|
63
|
-
"date",
|
|
64
|
-
"date of birth",
|
|
65
|
-
"bank account number",
|
|
66
|
-
"iban",
|
|
67
|
-
"tax identification number",
|
|
68
|
-
"identity card number",
|
|
69
|
-
"birth number",
|
|
70
|
-
"national identification number",
|
|
71
|
-
"social security number",
|
|
72
|
-
"registration number",
|
|
73
|
-
"credit card number",
|
|
74
|
-
"passport number",
|
|
75
|
-
"monetary amount",
|
|
76
|
-
"land parcel",
|
|
77
|
-
"misc"
|
|
78
|
-
];
|
|
79
18
|
const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
80
19
|
//#endregion
|
|
81
20
|
//#region src/context.ts
|
|
@@ -948,6 +887,7 @@ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTE
|
|
|
948
887
|
const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
|
|
949
888
|
const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
|
|
950
889
|
const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
|
|
890
|
+
const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
|
|
951
891
|
const ALLCAP_WORD = `[${UPPER}]{2,}`;
|
|
952
892
|
const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
|
|
953
893
|
let legalRoleHeadsCache = null;
|
|
@@ -975,11 +915,22 @@ const warmLegalRoleHeads = async () => {
|
|
|
975
915
|
loadLegalRoleHeads(),
|
|
976
916
|
loadAllLegalSuffixes(),
|
|
977
917
|
loadSentenceVerbIndicators(),
|
|
978
|
-
loadClauseNounHeads()
|
|
918
|
+
loadClauseNounHeads(),
|
|
919
|
+
loadConnectorProseHeads(),
|
|
920
|
+
loadStructuralSingleCapPrefixes()
|
|
979
921
|
]);
|
|
980
922
|
};
|
|
981
923
|
let allLegalSuffixesCache = null;
|
|
982
924
|
let allLegalSuffixesPromise = null;
|
|
925
|
+
let normalizedLegalBoundarySuffixesCache = null;
|
|
926
|
+
let normalizedInNameLegalFormWordsCache = null;
|
|
927
|
+
const normalizeLegalSuffixToken = (suffix) => suffix.replace(/[.,\s]/g, "");
|
|
928
|
+
const isBoundaryLegalSuffixForm = (form) => {
|
|
929
|
+
const normalized = normalizeLegalSuffixToken(form);
|
|
930
|
+
if (normalized.length === 0) return false;
|
|
931
|
+
if (LEGAL_SUFFIXES.includes(form)) return true;
|
|
932
|
+
return /[.]/u.test(form) || normalized === normalized.toUpperCase();
|
|
933
|
+
};
|
|
983
934
|
const loadAllLegalSuffixes = async () => {
|
|
984
935
|
if (allLegalSuffixesCache) return allLegalSuffixesCache;
|
|
985
936
|
if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
|
|
@@ -1004,11 +955,15 @@ const loadAllLegalSuffixes = async () => {
|
|
|
1004
955
|
}
|
|
1005
956
|
out.sort((a, b) => b.length - a.length);
|
|
1006
957
|
allLegalSuffixesCache = out;
|
|
958
|
+
normalizedLegalBoundarySuffixesCache = new Set(out.filter(isBoundaryLegalSuffixForm).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
959
|
+
normalizedInNameLegalFormWordsCache = new Set(out.filter((form) => !isBoundaryLegalSuffixForm(form) && !/\s/u.test(form)).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
1007
960
|
return out;
|
|
1008
961
|
})();
|
|
1009
962
|
return allLegalSuffixesPromise;
|
|
1010
963
|
};
|
|
1011
964
|
const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
|
|
965
|
+
const getNormalizedLegalBoundarySuffixesSync = () => normalizedLegalBoundarySuffixesCache ?? new Set(LEGAL_SUFFIXES.map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
|
|
966
|
+
const getNormalizedInNameLegalFormWordsSync = () => normalizedInNameLegalFormWordsCache ?? /* @__PURE__ */ new Set();
|
|
1012
967
|
/**
|
|
1013
968
|
* Sync accessor for the full legal-form vocabulary
|
|
1014
969
|
* (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
|
|
@@ -1048,18 +1003,72 @@ const loadClauseNounHeads = async () => {
|
|
|
1048
1003
|
return clauseNounHeadsPromise;
|
|
1049
1004
|
};
|
|
1050
1005
|
const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
|
|
1006
|
+
let connectorProseHeadsCache = null;
|
|
1007
|
+
let connectorProseHeadsPromise = null;
|
|
1008
|
+
const loadConnectorProseHeads = async () => {
|
|
1009
|
+
if (connectorProseHeadsCache) return connectorProseHeadsCache;
|
|
1010
|
+
if (connectorProseHeadsPromise) return connectorProseHeadsPromise;
|
|
1011
|
+
connectorProseHeadsPromise = (async () => {
|
|
1012
|
+
let data = {};
|
|
1013
|
+
try {
|
|
1014
|
+
const mod = await import("./generic-roles.mjs");
|
|
1015
|
+
data = mod.default ?? mod;
|
|
1016
|
+
} catch (err) {
|
|
1017
|
+
console.warn("[anonymize] legal-forms: failed to load generic-roles.json:", err);
|
|
1018
|
+
}
|
|
1019
|
+
const all = /* @__PURE__ */ new Set();
|
|
1020
|
+
if (Array.isArray(data.roles)) {
|
|
1021
|
+
for (const role of data.roles) if (typeof role === "string" && role.length > 0) all.add(role.toLowerCase());
|
|
1022
|
+
}
|
|
1023
|
+
connectorProseHeadsCache = all;
|
|
1024
|
+
return all;
|
|
1025
|
+
})();
|
|
1026
|
+
return connectorProseHeadsPromise;
|
|
1027
|
+
};
|
|
1028
|
+
const getConnectorProseHeadsSync = () => connectorProseHeadsCache ?? /* @__PURE__ */ new Set();
|
|
1029
|
+
let structuralSingleCapPrefixesCache = null;
|
|
1030
|
+
let structuralSingleCapPrefixesPromise = null;
|
|
1031
|
+
const loadStructuralSingleCapPrefixes = async () => {
|
|
1032
|
+
if (structuralSingleCapPrefixesCache) return structuralSingleCapPrefixesCache;
|
|
1033
|
+
if (structuralSingleCapPrefixesPromise) return structuralSingleCapPrefixesPromise;
|
|
1034
|
+
structuralSingleCapPrefixesPromise = (async () => {
|
|
1035
|
+
let data = {};
|
|
1036
|
+
try {
|
|
1037
|
+
const mod = await import("./structural-single-cap-prefixes.mjs");
|
|
1038
|
+
data = mod.default ?? mod;
|
|
1039
|
+
} catch (err) {
|
|
1040
|
+
console.warn("[anonymize] legal-forms: failed to load structural-single-cap-prefixes.json:", err);
|
|
1041
|
+
}
|
|
1042
|
+
const all = /* @__PURE__ */ new Set();
|
|
1043
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1044
|
+
if (key.startsWith("_")) continue;
|
|
1045
|
+
if (!Array.isArray(value)) continue;
|
|
1046
|
+
for (const prefix of value) {
|
|
1047
|
+
if (typeof prefix !== "string" || prefix.length === 0) continue;
|
|
1048
|
+
all.add(prefix.toLowerCase());
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
structuralSingleCapPrefixesCache = all;
|
|
1052
|
+
return all;
|
|
1053
|
+
})();
|
|
1054
|
+
return structuralSingleCapPrefixesPromise;
|
|
1055
|
+
};
|
|
1056
|
+
const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
|
|
1051
1057
|
const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
|
|
1052
1058
|
const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
|
|
1059
|
+
const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1053
1060
|
const buildPatternString = (forms) => {
|
|
1054
1061
|
if (forms.length === 0) return null;
|
|
1055
1062
|
const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1056
1063
|
const HSPACE = "[^\\S\\n]";
|
|
1057
1064
|
const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
|
|
1058
1065
|
const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
|
|
1059
|
-
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
|
|
1066
|
+
const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
|
|
1060
1067
|
const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
|
|
1061
1068
|
const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
|
|
1062
|
-
|
|
1069
|
+
const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
|
|
1070
|
+
const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
|
|
1071
|
+
return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
|
|
1063
1072
|
};
|
|
1064
1073
|
/**
|
|
1065
1074
|
* Build legal form regex pattern strings.
|
|
@@ -1092,6 +1101,7 @@ const buildLegalFormPatterns = async () => {
|
|
|
1092
1101
|
const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
|
|
1093
1102
|
const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
|
|
1094
1103
|
patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
|
|
1104
|
+
patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
|
|
1095
1105
|
return patterns;
|
|
1096
1106
|
};
|
|
1097
1107
|
const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
|
|
@@ -1101,6 +1111,12 @@ const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Ho
|
|
|
1101
1111
|
const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
|
|
1102
1112
|
const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
|
|
1103
1113
|
const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
|
|
1114
|
+
const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:[ \\t]+|,[ \\t]*)`, "u");
|
|
1115
|
+
const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
|
|
1116
|
+
const isStructuralSingleCapMatch = (text) => {
|
|
1117
|
+
const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
|
|
1118
|
+
return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
|
|
1119
|
+
};
|
|
1104
1120
|
/**
|
|
1105
1121
|
* Find the word ending just before `pos` in `text`,
|
|
1106
1122
|
* skipping any whitespace (not newlines).
|
|
@@ -1125,25 +1141,145 @@ const findWordBefore = (text, pos) => {
|
|
|
1125
1141
|
start: wordStart
|
|
1126
1142
|
};
|
|
1127
1143
|
};
|
|
1144
|
+
const hasSingleCapPrefixBefore = (fullText, matchStart) => {
|
|
1145
|
+
const prev = findWordBefore(fullText, matchStart);
|
|
1146
|
+
return prev !== null && prev.word.length === 1 && UPPER_LETTER_RE.test(prev.word);
|
|
1147
|
+
};
|
|
1148
|
+
const isBareSingleCapStructuralInnerMatch = (fullText, matchStart, text) => {
|
|
1149
|
+
if (!BARE_SINGLE_CAP_LEGAL_FORM_RE.test(text)) return false;
|
|
1150
|
+
const prev = findWordBefore(fullText, matchStart);
|
|
1151
|
+
return prev !== null && getStructuralSingleCapPrefixesSync().has(prev.word.toLowerCase());
|
|
1152
|
+
};
|
|
1153
|
+
const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
|
|
1154
|
+
let cut = -1;
|
|
1155
|
+
for (const suffix of getAllLegalSuffixesSync()) {
|
|
1156
|
+
const suffixClean = suffix.replace(/[.,\s]/g, "");
|
|
1157
|
+
if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
|
|
1158
|
+
let fromIndex = 0;
|
|
1159
|
+
while (fromIndex < entityText.length) {
|
|
1160
|
+
const suffixStart = entityText.indexOf(suffix, fromIndex);
|
|
1161
|
+
if (suffixStart === -1) break;
|
|
1162
|
+
fromIndex = suffixStart + suffix.length;
|
|
1163
|
+
const suffixEnd = suffixStart + suffix.length;
|
|
1164
|
+
if (suffixEnd >= entityText.length - 1) continue;
|
|
1165
|
+
const afterSuffix = entityText.slice(suffixEnd);
|
|
1166
|
+
const boundary = /^,\s+(?=\p{Lu})/u.exec(afterSuffix);
|
|
1167
|
+
if (boundary === null) continue;
|
|
1168
|
+
const nextStart = suffixEnd + boundary[0].length;
|
|
1169
|
+
const remainder = entityText.slice(nextStart);
|
|
1170
|
+
if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
|
|
1171
|
+
cut = Math.max(cut, nextStart);
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
if (cut <= 0) return {
|
|
1175
|
+
entityStart,
|
|
1176
|
+
entityText
|
|
1177
|
+
};
|
|
1178
|
+
return {
|
|
1179
|
+
entityStart: entityStart + cut,
|
|
1180
|
+
entityText: entityText.slice(cut)
|
|
1181
|
+
};
|
|
1182
|
+
};
|
|
1183
|
+
const hasMiddleInitialBefore = (fullText, pos) => {
|
|
1184
|
+
const previousWord = findWordBefore(fullText, pos);
|
|
1185
|
+
if (!previousWord) return false;
|
|
1186
|
+
let scan = previousWord.start - 1;
|
|
1187
|
+
while (scan >= 0 && (fullText[scan] === " " || fullText[scan] === " ")) scan--;
|
|
1188
|
+
return scan >= 1 && fullText[scan] === "." && UPPER_LETTER_RE.test(fullText[scan - 1] ?? "");
|
|
1189
|
+
};
|
|
1128
1190
|
/**
|
|
1129
1191
|
* Count consecutive uppercase-starting words immediately
|
|
1130
1192
|
* before `pos`. Stops at the first non-upper word or at
|
|
1131
|
-
* text/line start. Used to disambiguate
|
|
1132
|
-
* and <ORG>"
|
|
1133
|
-
|
|
1134
|
-
|
|
1193
|
+
* text/line start. Used to disambiguate sentence prose
|
|
1194
|
+
* ("<First> <Last> and <ORG>", "<Defined-Term> and
|
|
1195
|
+
* <ORG>") from multi-word organisation names that span
|
|
1196
|
+
* an "and" connector ("UniCredit Bank Czech Republic and
|
|
1197
|
+
* Slovakia, a.s.").
|
|
1198
|
+
*
|
|
1199
|
+
* When `crossInNamePreps` is true, the walk also steps
|
|
1200
|
+
* over in-name lowercase prepositions ("of", "the") as
|
|
1201
|
+
* long as they sit between two upper words. This lets
|
|
1202
|
+
* the suffix-mode "and"-crossing logic see through
|
|
1203
|
+
* "<Trust ← and ← America ← of ← Bank>" and emit one
|
|
1204
|
+
* full organisation span.
|
|
1205
|
+
*/
|
|
1206
|
+
const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
|
|
1135
1207
|
let count = 0;
|
|
1136
1208
|
let scan = pos;
|
|
1137
1209
|
while (scan > 0) {
|
|
1138
1210
|
const found = findWordBefore(fullText, scan);
|
|
1139
|
-
if (
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1211
|
+
if (found) {
|
|
1212
|
+
if (UPPER_LETTER_RE.test(found.word)) {
|
|
1213
|
+
count++;
|
|
1214
|
+
scan = found.start;
|
|
1215
|
+
continue;
|
|
1216
|
+
}
|
|
1217
|
+
if (crossInNamePreps && IN_NAME_PREPOSITION_RE.test(found.word)) {
|
|
1218
|
+
const prev = findWordBefore(fullText, found.start);
|
|
1219
|
+
if (!prev) break;
|
|
1220
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1221
|
+
scan = found.start;
|
|
1222
|
+
continue;
|
|
1223
|
+
}
|
|
1224
|
+
break;
|
|
1225
|
+
}
|
|
1226
|
+
let p = scan - 1;
|
|
1227
|
+
while (p >= 0 && (fullText[p] === " " || fullText[p] === " ")) p--;
|
|
1228
|
+
if (p >= 1 && fullText[p] === "." && UPPER_LETTER_RE.test(fullText[p - 1] ?? "")) {
|
|
1229
|
+
count++;
|
|
1230
|
+
scan = p - 1;
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
break;
|
|
1143
1234
|
}
|
|
1144
1235
|
return count;
|
|
1145
1236
|
};
|
|
1146
1237
|
/**
|
|
1238
|
+
* True when `word` is a recognized legal-form suffix
|
|
1239
|
+
* (case-sensitive against the legal-forms vocabulary).
|
|
1240
|
+
* Used when deciding whether to cross an "and" connector
|
|
1241
|
+
* during backward extension — if the word immediately
|
|
1242
|
+
* preceding the connector is itself a legal-form suffix,
|
|
1243
|
+
* the "and" sits between two organisation names rather
|
|
1244
|
+
* than inside one ("Morgan Securities LLC and Allen &
|
|
1245
|
+
* Company LLC"), so the walk must stop there.
|
|
1246
|
+
*/
|
|
1247
|
+
const isKnownLegalFormSuffix = (word) => {
|
|
1248
|
+
if (word.length === 0) return false;
|
|
1249
|
+
return getNormalizedLegalBoundarySuffixesSync().has(word);
|
|
1250
|
+
};
|
|
1251
|
+
const isInNameLegalFormWord = (word) => {
|
|
1252
|
+
if (word.length === 0) return false;
|
|
1253
|
+
return getNormalizedInNameLegalFormWordsSync().has(word);
|
|
1254
|
+
};
|
|
1255
|
+
/**
|
|
1256
|
+
* If `pos` is immediately preceded (modulo horizontal
|
|
1257
|
+
* whitespace) by an initial-dot run like `J.P.`, `U.S.`,
|
|
1258
|
+
* or `N.A.`, return the position where the initial run
|
|
1259
|
+
* starts. Otherwise return `pos` unchanged. The run must
|
|
1260
|
+
* be word-bounded on the left so we never absorb a stray
|
|
1261
|
+
* sentence-ending dot.
|
|
1262
|
+
*/
|
|
1263
|
+
const skipInitialsBackward = (fullText, pos) => {
|
|
1264
|
+
let scan = pos - 1;
|
|
1265
|
+
while (scan >= 0) {
|
|
1266
|
+
const ch = fullText.charAt(scan);
|
|
1267
|
+
if (ch === "\n" || !/\s/.test(ch)) break;
|
|
1268
|
+
scan--;
|
|
1269
|
+
}
|
|
1270
|
+
if (scan < 0 || fullText.charAt(scan) !== ".") return pos;
|
|
1271
|
+
const scanLimit = Math.max(0, scan + 1 - 100);
|
|
1272
|
+
const head = fullText.slice(scanLimit, scan + 1);
|
|
1273
|
+
const match = /(?:\p{Lu}\.[^\S\n]?){2,}$/u.exec(head);
|
|
1274
|
+
if (match === null) return pos;
|
|
1275
|
+
const start = scanLimit + match.index;
|
|
1276
|
+
if (start > 0) {
|
|
1277
|
+
const prevCh = fullText.charAt(start - 1);
|
|
1278
|
+
if (/[\p{L}\p{M}\p{N}]/u.test(prevCh)) return pos;
|
|
1279
|
+
}
|
|
1280
|
+
return start;
|
|
1281
|
+
};
|
|
1282
|
+
/**
|
|
1147
1283
|
* Extend a match backward through uppercase words and
|
|
1148
1284
|
* lowercase connectors. Stops at start of text,
|
|
1149
1285
|
* newline, or a word that doesn't qualify.
|
|
@@ -1172,19 +1308,30 @@ const extendBackward = (fullText, matchStart, options = {}) => {
|
|
|
1172
1308
|
const isConnector = CONNECTOR_RE.test(word);
|
|
1173
1309
|
const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
|
|
1174
1310
|
if (isUpper) pos = wordStart;
|
|
1175
|
-
else if (isConnector) {
|
|
1176
|
-
if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
|
|
1311
|
+
else if (isConnector) if (AND_TYPE_CONNECTOR_RE.test(word)) {
|
|
1177
1312
|
const prev = findWordBefore(fullText, wordStart);
|
|
1178
1313
|
if (!prev) break;
|
|
1179
1314
|
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1315
|
+
if (isKnownLegalFormSuffix(prev.word)) break;
|
|
1316
|
+
const upperWordsBefore = countUpperWordsBefore(fullText, wordStart, suffixMode);
|
|
1317
|
+
const middleInitialBefore = hasMiddleInitialBefore(fullText, wordStart);
|
|
1318
|
+
if (upperWordsBefore <= 1 && (getClauseNounHeadsSync().has(prev.word.toLowerCase()) || getConnectorProseHeadsSync().has(prev.word.toLowerCase()))) break;
|
|
1319
|
+
if (suffixMode ? middleInitialBefore && hasSingleCapPrefixBefore(fullText, matchStart) : upperWordsBefore === 2 && !isInNameLegalFormWord(prev.word) || middleInitialBefore) break;
|
|
1180
1320
|
pos = prev.start;
|
|
1181
|
-
} else
|
|
1321
|
+
} else {
|
|
1322
|
+
const prev = findWordBefore(fullText, wordStart);
|
|
1323
|
+
if (!prev) break;
|
|
1324
|
+
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1325
|
+
pos = prev.start;
|
|
1326
|
+
}
|
|
1327
|
+
else if (isInNamePrep) {
|
|
1182
1328
|
const prev = findWordBefore(fullText, wordStart);
|
|
1183
1329
|
if (!prev) break;
|
|
1184
1330
|
if (!UPPER_LETTER_RE.test(prev.word)) break;
|
|
1185
1331
|
pos = prev.start;
|
|
1186
1332
|
} else break;
|
|
1187
1333
|
}
|
|
1334
|
+
pos = skipInitialsBackward(fullText, pos);
|
|
1188
1335
|
return pos;
|
|
1189
1336
|
};
|
|
1190
1337
|
const trimLeadingClause = (text) => {
|
|
@@ -1225,6 +1372,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1225
1372
|
const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
|
|
1226
1373
|
let processedStart = match.start;
|
|
1227
1374
|
let processedText = text;
|
|
1375
|
+
if (isStructuralSingleCapMatch(processedText) || fullText !== void 0 && isBareSingleCapStructuralInnerMatch(fullText, match.start, text)) continue;
|
|
1228
1376
|
let trimmed = false;
|
|
1229
1377
|
const firstWordText = firstWordMatch?.[0] ?? "";
|
|
1230
1378
|
const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
|
|
@@ -1275,12 +1423,15 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
|
|
|
1275
1423
|
let entityStart = processedStart;
|
|
1276
1424
|
let entityText = processedText;
|
|
1277
1425
|
if (fullText && !trimmed) {
|
|
1278
|
-
const extended = extendBackward(fullText, processedStart);
|
|
1426
|
+
const extended = !BARE_SINGLE_CAP_LEGAL_FORM_RE.test(processedText) ? extendBackward(fullText, processedStart) : processedStart;
|
|
1279
1427
|
if (extended < processedStart) {
|
|
1280
1428
|
entityStart = extended;
|
|
1281
1429
|
entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
|
|
1282
1430
|
}
|
|
1283
1431
|
}
|
|
1432
|
+
const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
|
|
1433
|
+
entityStart = listTrim.entityStart;
|
|
1434
|
+
entityText = listTrim.entityText;
|
|
1284
1435
|
const clauseTrim = trimLeadingClause(entityText);
|
|
1285
1436
|
if (clauseTrim.offset > 0) {
|
|
1286
1437
|
entityStart += clauseTrim.offset;
|
|
@@ -1627,20 +1778,23 @@ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, dat
|
|
|
1627
1778
|
const label = data.labels[localIdx];
|
|
1628
1779
|
if (!label) continue;
|
|
1629
1780
|
const extended = tryPrefixExtension(fullText, match.start, match.end);
|
|
1781
|
+
const isExtended = extended !== null;
|
|
1630
1782
|
const end = extended?.end ?? match.end;
|
|
1631
1783
|
const text = extended?.text ?? fullText.slice(match.start, match.end);
|
|
1632
1784
|
exactSpans.push({
|
|
1633
1785
|
start: match.start,
|
|
1634
1786
|
end
|
|
1635
1787
|
});
|
|
1636
|
-
|
|
1788
|
+
const entity = {
|
|
1637
1789
|
start: match.start,
|
|
1638
1790
|
end,
|
|
1639
1791
|
label,
|
|
1640
1792
|
text,
|
|
1641
1793
|
score: .9,
|
|
1642
1794
|
source: DETECTION_SOURCES.GAZETTEER
|
|
1643
|
-
}
|
|
1795
|
+
};
|
|
1796
|
+
if (isExtended) entity.sourceDetail = "gazetteer-extension";
|
|
1797
|
+
results.push(entity);
|
|
1644
1798
|
}
|
|
1645
1799
|
for (const match of allMatches) {
|
|
1646
1800
|
const idx = match.pattern;
|
|
@@ -2174,7 +2328,9 @@ const POST_NOMINALS = [
|
|
|
2174
2328
|
"CIPM",
|
|
2175
2329
|
"CIPT",
|
|
2176
2330
|
"CIPP/E",
|
|
2177
|
-
"CIPP"
|
|
2331
|
+
"CIPP",
|
|
2332
|
+
"KC",
|
|
2333
|
+
"QC"
|
|
2178
2334
|
];
|
|
2179
2335
|
//#endregion
|
|
2180
2336
|
//#region src/detectors/regex.ts
|
|
@@ -2183,8 +2339,10 @@ const MIN_MONTH_NAME_LENGTH = 3;
|
|
|
2183
2339
|
const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
|
|
2184
2340
|
/** Escape for use inside a regex alternation. */
|
|
2185
2341
|
const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2342
|
+
const escapeRegexPhrase = (s) => escapeRegex$2(s.trim()).replace(/\s+/g, "[^\\S\\n\\t]+");
|
|
2186
2343
|
/** Escape for use inside a regex character class. */
|
|
2187
2344
|
const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
|
|
2345
|
+
const toSortedAlternation = (values) => [...new Set(values.map(escapeRegexPhrase).filter((value) => value.length > 0))].toSorted((a, b) => b.length - a.length).join("|");
|
|
2188
2346
|
const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
2189
2347
|
const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
|
|
2190
2348
|
const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
|
|
@@ -2195,6 +2353,7 @@ const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).ma
|
|
|
2195
2353
|
const escaped = escapeRegex$2(h);
|
|
2196
2354
|
return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
|
|
2197
2355
|
}).join("|");
|
|
2356
|
+
const AMOUNT_WORDS = amount_words_default;
|
|
2198
2357
|
const toEntry = (validator, label, score) => {
|
|
2199
2358
|
const pattern = toRegex(validator).source;
|
|
2200
2359
|
if (!pattern) return null;
|
|
@@ -2277,6 +2436,11 @@ const HONORIFIC_PERSON = {
|
|
|
2277
2436
|
label: "person",
|
|
2278
2437
|
score: .95
|
|
2279
2438
|
};
|
|
2439
|
+
const POSTNOMINAL_PERSON = {
|
|
2440
|
+
pattern: `${NAME_WORD}(?:(?:${SP}|-){1,2}(?:${PARTICLE}${SP}+)?${NAME_WORD}){1,3},?${SP}+(?:KC|QC)\\b`,
|
|
2441
|
+
label: "person",
|
|
2442
|
+
score: .95
|
|
2443
|
+
};
|
|
2280
2444
|
const IBAN = {
|
|
2281
2445
|
pattern: "\\b[A-Z]{2}\\d{2}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{4}\\s?[\\dA-Z]{0,14}\\b",
|
|
2282
2446
|
label: "iban",
|
|
@@ -2403,6 +2567,58 @@ const SHORT_TLDS = "de|at|be|se|fi|dk|no|it|uk";
|
|
|
2403
2567
|
const HOST_LABEL = `[a-zA-Z0-9](?:[a-zA-Z0-9\\-]*[a-zA-Z0-9])?`;
|
|
2404
2568
|
const BARE_HOST = `\\b[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]`;
|
|
2405
2569
|
const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
|
|
2570
|
+
const BARE_DOMAIN = {
|
|
2571
|
+
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
2572
|
+
label: "url",
|
|
2573
|
+
score: .9
|
|
2574
|
+
};
|
|
2575
|
+
const IPV6_ADDRESS = {
|
|
2576
|
+
pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
|
|
2577
|
+
label: "ip address",
|
|
2578
|
+
score: 1
|
|
2579
|
+
};
|
|
2580
|
+
const MAC_ADDRESS = {
|
|
2581
|
+
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
2582
|
+
label: "mac address",
|
|
2583
|
+
score: 1
|
|
2584
|
+
};
|
|
2585
|
+
const UK_POSTCODE = {
|
|
2586
|
+
pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
|
|
2587
|
+
label: "address",
|
|
2588
|
+
score: .9
|
|
2589
|
+
};
|
|
2590
|
+
const UK_NINO = {
|
|
2591
|
+
pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
|
|
2592
|
+
label: "social security number",
|
|
2593
|
+
score: .95,
|
|
2594
|
+
validator: gb.nino
|
|
2595
|
+
};
|
|
2596
|
+
const TIME_12H = {
|
|
2597
|
+
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
2598
|
+
label: "date",
|
|
2599
|
+
score: .9
|
|
2600
|
+
};
|
|
2601
|
+
const PERCENT_TOKEN = `${`(?:[+${DASH_INNER}])?(?:\\d{1,3}(?:[.,]\\d{3})+(?:[.,]\\d{1,4})?|\\d+(?:[.,]\\d{1,4})?)`}[^\\S\\n]{0,2}%`;
|
|
2602
|
+
const PERCENT_RANGE_NUMBER = `\\d+(?:[.,]\\d{1,4})?`;
|
|
2603
|
+
const PERCENT_RANGE = `${PERCENT_RANGE_NUMBER}[^\\S\\n]*${DASH}[^\\S\\n]*${PERCENT_RANGE_NUMBER}[^\\S\\n]{0,2}%`;
|
|
2604
|
+
const buildPercentWordPattern = (config) => {
|
|
2605
|
+
const phrases = [];
|
|
2606
|
+
for (const entry of config.percentages ?? []) {
|
|
2607
|
+
const ones = entry.ones.map(escapeRegex$2);
|
|
2608
|
+
const standalone = (entry.standalone ?? []).map(escapeRegex$2);
|
|
2609
|
+
const baseWords = [
|
|
2610
|
+
...ones,
|
|
2611
|
+
...entry.teens,
|
|
2612
|
+
...entry.tens
|
|
2613
|
+
].map(escapeRegex$2);
|
|
2614
|
+
const compoundSeparator = entry.allowSpaceCompoundSeparator ? `(?:${DASH}|[^\\S\\n]+)` : DASH;
|
|
2615
|
+
const compound = `(?:${baseWords.join("|")})` + (ones.length > 0 ? `(?:${compoundSeparator}(?:${ones.join("|")}))?` : "");
|
|
2616
|
+
const word = `(?:${[...standalone, compound].join("|")})`;
|
|
2617
|
+
const keyword = `(?:${entry.keywords.map(escapeRegex$2).join("|")})`;
|
|
2618
|
+
phrases.push(`${word}[^\\S\\n]+${keyword}`);
|
|
2619
|
+
}
|
|
2620
|
+
return phrases.length > 0 ? `(?i:(?:${phrases.join("|")}))` : "(?!)";
|
|
2621
|
+
};
|
|
2406
2622
|
/**
|
|
2407
2623
|
* All static PII regex definitions. Scanned in a
|
|
2408
2624
|
* single pass by @stll/regex-set (Rust DFA).
|
|
@@ -2421,6 +2637,7 @@ const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
|
|
|
2421
2637
|
const ALL_REGEX_DEFS = [
|
|
2422
2638
|
TITLED_PERSON,
|
|
2423
2639
|
HONORIFIC_PERSON,
|
|
2640
|
+
POSTNOMINAL_PERSON,
|
|
2424
2641
|
IBAN,
|
|
2425
2642
|
EMAIL,
|
|
2426
2643
|
INTL_PHONE,
|
|
@@ -2443,25 +2660,16 @@ const ALL_REGEX_DEFS = [
|
|
|
2443
2660
|
BR_RG_WITH_SSP,
|
|
2444
2661
|
BR_OAB,
|
|
2445
2662
|
URL,
|
|
2663
|
+
IPV6_ADDRESS,
|
|
2664
|
+
MAC_ADDRESS,
|
|
2665
|
+
BARE_DOMAIN,
|
|
2666
|
+
UK_POSTCODE,
|
|
2667
|
+
UK_NINO,
|
|
2668
|
+
TIME_12H,
|
|
2446
2669
|
{
|
|
2447
|
-
pattern:
|
|
2448
|
-
label: "
|
|
2449
|
-
score:
|
|
2450
|
-
},
|
|
2451
|
-
{
|
|
2452
|
-
pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
|
|
2453
|
-
label: "mac address",
|
|
2454
|
-
score: 1
|
|
2455
|
-
},
|
|
2456
|
-
{
|
|
2457
|
-
pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
|
|
2458
|
-
label: "url",
|
|
2459
|
-
score: .9
|
|
2460
|
-
},
|
|
2461
|
-
{
|
|
2462
|
-
pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
|
|
2463
|
-
label: "date",
|
|
2464
|
-
score: .9
|
|
2670
|
+
pattern: `(?<![\\p{L}\\p{N}_.,])(?:${buildPercentWordPattern(AMOUNT_WORDS)}[^\\S\\n]*\\([^\\S\\n]*${PERCENT_TOKEN}[^\\S\\n]*\\)|${PERCENT_RANGE}|${PERCENT_TOKEN})(?![\\p{L}\\p{N}_])`,
|
|
2671
|
+
label: "monetary amount",
|
|
2672
|
+
score: .85
|
|
2465
2673
|
},
|
|
2466
2674
|
...STDNUM_ENTRIES
|
|
2467
2675
|
];
|
|
@@ -2535,6 +2743,41 @@ const DATE_PATTERN_META = Object.freeze({
|
|
|
2535
2743
|
label: "date",
|
|
2536
2744
|
score: 1
|
|
2537
2745
|
});
|
|
2746
|
+
const buildMagnitudePattern = (config) => {
|
|
2747
|
+
const words = [];
|
|
2748
|
+
const caseInsensitiveAbbreviations = [];
|
|
2749
|
+
const caseSensitiveAbbreviations = [];
|
|
2750
|
+
for (const entry of config.magnitudeSuffixes ?? []) {
|
|
2751
|
+
words.push(...entry.words ?? []);
|
|
2752
|
+
caseInsensitiveAbbreviations.push(...entry.abbreviationsCaseInsensitive ?? []);
|
|
2753
|
+
caseSensitiveAbbreviations.push(...entry.abbreviationsCaseSensitive ?? []);
|
|
2754
|
+
}
|
|
2755
|
+
const branches = [];
|
|
2756
|
+
const wordsAlt = toSortedAlternation(words);
|
|
2757
|
+
const abbreviationCiAlt = toSortedAlternation(caseInsensitiveAbbreviations);
|
|
2758
|
+
const abbreviationCsAlt = toSortedAlternation(caseSensitiveAbbreviations);
|
|
2759
|
+
if (wordsAlt) branches.push(`[^\\S\\n\\t]+(?i:(?:${wordsAlt}))\\b`);
|
|
2760
|
+
if (abbreviationCiAlt) branches.push(`[^\\S\\n\\t]?(?i:${abbreviationCiAlt})\\b`);
|
|
2761
|
+
if (abbreviationCsAlt) branches.push(`[^\\S\\n\\t]?(?:${abbreviationCsAlt})\\b`);
|
|
2762
|
+
return branches.length > 0 ? `(?:${branches.join("|")})?` : "";
|
|
2763
|
+
};
|
|
2764
|
+
const buildQuantityFollowerGuard = (config) => {
|
|
2765
|
+
const modifiers = [];
|
|
2766
|
+
const nouns = [];
|
|
2767
|
+
for (const entry of config.shareQuantityTerms ?? []) {
|
|
2768
|
+
modifiers.push(...entry.modifiers ?? []);
|
|
2769
|
+
nouns.push(...entry.nouns);
|
|
2770
|
+
}
|
|
2771
|
+
const modifierAlt = toSortedAlternation(modifiers);
|
|
2772
|
+
const nounAlt = toSortedAlternation(nouns);
|
|
2773
|
+
if (!nounAlt) return "";
|
|
2774
|
+
return `(?![^\\S\\n\\t]+(?i:${modifierAlt ? `(?:(?:${modifierAlt})[^\\S\\n\\t]+){0,3}` : ""}(?:${nounAlt}))\\b)`;
|
|
2775
|
+
};
|
|
2776
|
+
const buildFinancialLexicons = (config) => Object.freeze({
|
|
2777
|
+
magnitude: buildMagnitudePattern(config),
|
|
2778
|
+
quantityFollowerGuard: buildQuantityFollowerGuard(config)
|
|
2779
|
+
});
|
|
2780
|
+
const FINANCIAL_LEXICONS = buildFinancialLexicons(AMOUNT_WORDS);
|
|
2538
2781
|
/**
|
|
2539
2782
|
* Build symbol character class, code alternation,
|
|
2540
2783
|
* and local-name alternation from currencies.json,
|
|
@@ -2574,9 +2817,13 @@ const buildCurrencyPatterns = (data) => {
|
|
|
2574
2817
|
const patterns = [];
|
|
2575
2818
|
const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
|
|
2576
2819
|
const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
|
|
2577
|
-
|
|
2578
|
-
if (
|
|
2579
|
-
if (trailingAlt) patterns.push(`\\b
|
|
2820
|
+
const MAGNITUDE = FINANCIAL_LEXICONS.magnitude;
|
|
2821
|
+
if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${MAGNITUDE}${END}`);
|
|
2822
|
+
if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${MAGNITUDE}${END}`);
|
|
2823
|
+
if (trailingAlt) {
|
|
2824
|
+
const optionalLeadingSymbol = symbols ? `(?<![\\p{L}\\p{N}_])(?:[${symbols}][^\\S\\n\\t]?)?` : "\\b";
|
|
2825
|
+
patterns.push(`${optionalLeadingSymbol}${NUM}${DECIMAL}${MAGNITUDE}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${FINANCIAL_LEXICONS.quantityFollowerGuard}${END}`);
|
|
2826
|
+
}
|
|
2580
2827
|
return patterns;
|
|
2581
2828
|
};
|
|
2582
2829
|
/** Cached promise for currency patterns. Loaded once. */
|
|
@@ -2890,10 +3137,19 @@ const stripQuotes = (value) => {
|
|
|
2890
3137
|
text: stripped
|
|
2891
3138
|
};
|
|
2892
3139
|
};
|
|
2893
|
-
/**
|
|
3140
|
+
/**
|
|
3141
|
+
* Hard stop characters for to-next-comma scanning. A closing
|
|
3142
|
+
* parenthesis or closing bracket terminates a clause just like
|
|
3143
|
+
* an opening one: `State of New York or any other jurisdiction)`
|
|
3144
|
+
* is the tail of a parenthesised insertion, not the start of a
|
|
3145
|
+
* larger phrase that should be absorbed into a jurisdiction span.
|
|
3146
|
+
*/
|
|
2894
3147
|
const COMMA_STOP_CHARS = new Set([
|
|
2895
3148
|
"\n",
|
|
2896
3149
|
"(",
|
|
3150
|
+
")",
|
|
3151
|
+
"[",
|
|
3152
|
+
"]",
|
|
2897
3153
|
" ",
|
|
2898
3154
|
";"
|
|
2899
3155
|
]);
|
|
@@ -3059,6 +3315,27 @@ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STO
|
|
|
3059
3315
|
const warmAddressStopKeywords = async () => {
|
|
3060
3316
|
await loadAddressStopKeywords();
|
|
3061
3317
|
};
|
|
3318
|
+
const MAX_TRIGGER_VALUE_LEN = 100;
|
|
3319
|
+
const MIN_TRIGGER_PHONE_DIGITS = 5;
|
|
3320
|
+
const PHONE_VALUE_START_RE = /^[+(\d]/;
|
|
3321
|
+
const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
|
|
3322
|
+
const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
3323
|
+
const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
3324
|
+
const capAtWordBoundary = (valueText, cap) => {
|
|
3325
|
+
let capped = cap;
|
|
3326
|
+
const isWordChar = (i) => /[\p{L}\p{N}]/u.test(valueText[i] ?? "");
|
|
3327
|
+
while (capped > 0 && isWordChar(capped - 1) && isWordChar(capped)) capped--;
|
|
3328
|
+
return capped;
|
|
3329
|
+
};
|
|
3330
|
+
const isPlausiblePhoneTriggerValue = (value) => {
|
|
3331
|
+
const trimmed = value.trimStart();
|
|
3332
|
+
if (!PHONE_VALUE_START_RE.test(trimmed)) return false;
|
|
3333
|
+
if (ISO_DATE_PREFIX_RE.test(trimmed)) return false;
|
|
3334
|
+
if (INLINE_FIELD_LABEL_RE.test(trimmed)) return false;
|
|
3335
|
+
let digits = 0;
|
|
3336
|
+
for (const ch of trimmed) if (/\d/.test(ch)) digits++;
|
|
3337
|
+
return digits >= MIN_TRIGGER_PHONE_DIGITS;
|
|
3338
|
+
};
|
|
3062
3339
|
const extractValue = (text, triggerEnd, strategy, label) => {
|
|
3063
3340
|
const remaining = text.slice(triggerEnd);
|
|
3064
3341
|
const stripped = remaining.replace(/^[\s:;]+/, "");
|
|
@@ -3070,21 +3347,11 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3070
3347
|
const stopWords = strategy.stopWords ?? [];
|
|
3071
3348
|
const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
|
|
3072
3349
|
let end = 0;
|
|
3073
|
-
let foundStop = false;
|
|
3074
3350
|
while (end < valueText.length) {
|
|
3075
3351
|
const ch = valueText[end];
|
|
3076
|
-
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch))
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
}
|
|
3080
|
-
if (ch === "." && isSentenceTerminator(valueText, end)) {
|
|
3081
|
-
foundStop = true;
|
|
3082
|
-
break;
|
|
3083
|
-
}
|
|
3084
|
-
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
|
|
3085
|
-
foundStop = true;
|
|
3086
|
-
break;
|
|
3087
|
-
}
|
|
3352
|
+
if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) break;
|
|
3353
|
+
if (ch === "." && isSentenceTerminator(valueText, end)) break;
|
|
3354
|
+
if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) break;
|
|
3088
3355
|
if (ch === ",") {
|
|
3089
3356
|
const afterComma = valueText.slice(end);
|
|
3090
3357
|
if (DECIMAL_COMMA_RE.test(afterComma)) {
|
|
@@ -3096,12 +3363,12 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3096
3363
|
end += degreeMatch[0].length;
|
|
3097
3364
|
continue;
|
|
3098
3365
|
}
|
|
3099
|
-
foundStop = true;
|
|
3100
3366
|
break;
|
|
3101
3367
|
}
|
|
3102
3368
|
end++;
|
|
3103
3369
|
}
|
|
3104
|
-
|
|
3370
|
+
const lengthCap = strategy.maxLength ?? 100;
|
|
3371
|
+
if (end > lengthCap) end = capAtWordBoundary(valueText, lengthCap);
|
|
3105
3372
|
const rawSlice = valueText.slice(0, end);
|
|
3106
3373
|
const extracted = rawSlice.trim();
|
|
3107
3374
|
if (extracted.length === 0) return null;
|
|
@@ -3115,12 +3382,24 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
3115
3382
|
case "to-end-of-line": {
|
|
3116
3383
|
const consumed = remaining.length - valueText.length;
|
|
3117
3384
|
if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
|
|
3118
|
-
const LINE_STOPS = ["\n"];
|
|
3385
|
+
const LINE_STOPS = ["\n", " "];
|
|
3119
3386
|
let end = valueText.length;
|
|
3387
|
+
let foundLineStop = false;
|
|
3120
3388
|
for (const ch of LINE_STOPS) {
|
|
3121
3389
|
const idx = valueText.indexOf(ch);
|
|
3122
|
-
if (idx !== -1 && idx < end)
|
|
3390
|
+
if (idx !== -1 && idx < end) {
|
|
3391
|
+
end = idx;
|
|
3392
|
+
foundLineStop = true;
|
|
3393
|
+
}
|
|
3123
3394
|
}
|
|
3395
|
+
if (label === "phone number") {
|
|
3396
|
+
const inlineLabel = INLINE_FIELD_LABEL_STOP_RE.exec(valueText.slice(0, end));
|
|
3397
|
+
if (inlineLabel) {
|
|
3398
|
+
end = inlineLabel.index;
|
|
3399
|
+
foundLineStop = true;
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
|
|
3124
3403
|
const rawSlice = valueText.slice(0, end);
|
|
3125
3404
|
const extracted = rawSlice.trim();
|
|
3126
3405
|
if (extracted.length === 0) return null;
|
|
@@ -3300,6 +3579,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
3300
3579
|
const value = rawValue ? stripQuotes(rawValue) : null;
|
|
3301
3580
|
if (value) {
|
|
3302
3581
|
if (!applyValidations(value.text, rule.validations)) continue;
|
|
3582
|
+
if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
|
|
3303
3583
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
3304
3584
|
const entityEnd = value.end;
|
|
3305
3585
|
const entityText = fullText.slice(entityStart, entityEnd);
|
|
@@ -3641,7 +3921,7 @@ const normalizeHomoglyphs = (text) => {
|
|
|
3641
3921
|
//#endregion
|
|
3642
3922
|
//#region src/filters/false-positives.ts
|
|
3643
3923
|
const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
|
|
3644
|
-
const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
|
|
3924
|
+
const POSTAL_CODE_RE$1 = /\d{3}\s?\d{2}/;
|
|
3645
3925
|
const HAS_DIGIT_RE = /\d/;
|
|
3646
3926
|
const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
|
|
3647
3927
|
const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
|
|
@@ -3706,7 +3986,7 @@ const normalizeEntity = (entity) => {
|
|
|
3706
3986
|
text
|
|
3707
3987
|
};
|
|
3708
3988
|
};
|
|
3709
|
-
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
3989
|
+
const EMPTY_GENERIC_ROLES$1 = /* @__PURE__ */ new Set();
|
|
3710
3990
|
/**
|
|
3711
3991
|
* Load generic-roles.json and cache the result on the
|
|
3712
3992
|
* given context. Must be awaited during pipeline init
|
|
@@ -3730,14 +4010,14 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3730
4010
|
return ctx.genericRolesPromise;
|
|
3731
4011
|
};
|
|
3732
4012
|
/** Sync accessor — returns empty set before init. */
|
|
3733
|
-
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
4013
|
+
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES$1;
|
|
3734
4014
|
const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
|
|
3735
4015
|
let _streetTypesRe = STREET_TYPES_SEED_RE;
|
|
3736
4016
|
let _streetTypesPromise = null;
|
|
3737
4017
|
const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3738
4018
|
const loadStreetTypeRegex = async () => {
|
|
3739
4019
|
try {
|
|
3740
|
-
const data = (await import("./address-street-types.mjs")).default ?? {};
|
|
4020
|
+
const data = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
|
|
3741
4021
|
const words = /* @__PURE__ */ new Set();
|
|
3742
4022
|
for (const [key, val] of Object.entries(data)) {
|
|
3743
4023
|
if (key.startsWith("_")) continue;
|
|
@@ -3806,7 +4086,7 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3806
4086
|
}
|
|
3807
4087
|
if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
|
|
3808
4088
|
if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
|
|
3809
|
-
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
4089
|
+
if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE$1.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3810
4090
|
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
|
|
3811
4091
|
if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
|
|
3812
4092
|
if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
|
|
@@ -3871,6 +4151,23 @@ const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
|
3871
4151
|
/** Sync accessor — returns empty set before init. */
|
|
3872
4152
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
3873
4153
|
/**
|
|
4154
|
+
* Curated dictionary entries that are pure dotted
|
|
4155
|
+
* single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
|
|
4156
|
+
* need targeted suffix guards. The AC search matches
|
|
4157
|
+
* case-insensitively on token boundaries where `.` is not
|
|
4158
|
+
* a word character, so `S.C.` can match inside `U.S.C.`.
|
|
4159
|
+
* Two-segment non-address aliases are too noisy and are
|
|
4160
|
+
* dropped at build time; longer official aliases stay
|
|
4161
|
+
* searchable and are only suppressed when the source text
|
|
4162
|
+
* shows they are the tail of a longer dotted token.
|
|
4163
|
+
* Caller-supplied custom entries are exempted.
|
|
4164
|
+
*/
|
|
4165
|
+
const DOTTED_ACRONYM_RE = /^(?=.{3,}$)\p{L}(?:\.\p{L}){0,3}\.?$/u;
|
|
4166
|
+
const isCuratedNoiseAcronym = (normalized) => DOTTED_ACRONYM_RE.test(normalized);
|
|
4167
|
+
const dottedAcronymSegmentCount = (normalized) => normalized.split(".").filter(Boolean).length;
|
|
4168
|
+
const isShortCuratedNoiseAcronym = (normalized) => isCuratedNoiseAcronym(normalized) && dottedAcronymSegmentCount(normalized) <= 2;
|
|
4169
|
+
const isDottedAcronymSuffixCollision = (fullText, start, matchText) => isCuratedNoiseAcronym(matchText) && /[\p{L}]\.$/u.test(fullText.slice(Math.max(0, start - 2), start));
|
|
4170
|
+
/**
|
|
3874
4171
|
* Common EU given names present in the stopwords-iso dataset
|
|
3875
4172
|
* but absent from the first-name corpus. Without this
|
|
3876
4173
|
* supplementary set, these names would pass through the
|
|
@@ -4006,7 +4303,7 @@ let streetTypeReLoaded = false;
|
|
|
4006
4303
|
const loadStreetTypeRe = async () => {
|
|
4007
4304
|
if (streetTypeReLoaded) return cachedStreetTypeRe;
|
|
4008
4305
|
try {
|
|
4009
|
-
const config = (await import("./address-street-types.mjs")).default ?? {};
|
|
4306
|
+
const config = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
|
|
4010
4307
|
const words = [];
|
|
4011
4308
|
for (const value of Object.values(config)) {
|
|
4012
4309
|
if (!Array.isArray(value)) continue;
|
|
@@ -4120,6 +4417,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
4120
4417
|
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
4121
4418
|
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
4122
4419
|
if (normalized.length === 0) return;
|
|
4420
|
+
if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
|
|
4123
4421
|
const lower = normalized.toLowerCase();
|
|
4124
4422
|
const existing = patternIndex.get(lower);
|
|
4125
4423
|
if (existing !== void 0) {
|
|
@@ -4197,6 +4495,7 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
4197
4495
|
const addNameEntry = (name, source) => {
|
|
4198
4496
|
const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
|
|
4199
4497
|
if (normalized.length === 0) return;
|
|
4498
|
+
if (isCuratedNoiseAcronym(normalized)) return;
|
|
4200
4499
|
const lower = normalized.toLowerCase();
|
|
4201
4500
|
const existing = patternIndex.get(lower);
|
|
4202
4501
|
if (existing !== void 0) {
|
|
@@ -4283,12 +4582,14 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
4283
4582
|
const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
|
|
4284
4583
|
const customLabels = customEdgesAreValid ? customPatternLabels : [];
|
|
4285
4584
|
if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
|
|
4286
|
-
const
|
|
4287
|
-
|
|
4585
|
+
const acronymMatchesAcronym = !(pattern.length > 0 && pattern.length <= 5 && ALL_UPPER_RE.test(pattern)) || ALL_UPPER_RE.test(matchText);
|
|
4586
|
+
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && acronymMatchesAcronym && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
|
|
4587
|
+
const filteredCuratedLabels = isDottedAcronymSuffixCollision(fullText, match.start, matchText) ? [] : curatedLabels;
|
|
4588
|
+
if (filteredCuratedLabels.length === 0 && customLabels.length === 0) continue;
|
|
4288
4589
|
const entry = {
|
|
4289
4590
|
start: match.start,
|
|
4290
4591
|
end: match.end,
|
|
4291
|
-
labels:
|
|
4592
|
+
labels: filteredCuratedLabels,
|
|
4292
4593
|
customLabels,
|
|
4293
4594
|
sources,
|
|
4294
4595
|
text: matchText,
|
|
@@ -4354,6 +4655,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
4354
4655
|
const first = chain.at(0);
|
|
4355
4656
|
const last = chain.at(-1);
|
|
4356
4657
|
if (!first || !last) continue;
|
|
4658
|
+
if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
|
|
4357
4659
|
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
4358
4660
|
const score = chain.length >= 2 ? .9 : .5;
|
|
4359
4661
|
if (chain.length === 1) {
|
|
@@ -4449,6 +4751,95 @@ const extendCityDistricts = (entities, fullText) => {
|
|
|
4449
4751
|
* by a capitalized word (for "Miroslav Braňka" when
|
|
4450
4752
|
* only "Braňka" matched).
|
|
4451
4753
|
*/
|
|
4754
|
+
/**
|
|
4755
|
+
* Defined-term marker: an opening typographic or straight
|
|
4756
|
+
* quote enclosing the chain start, AND a closing quote
|
|
4757
|
+
* within a short window followed by a
|
|
4758
|
+
* definitional cue (`means`, `shall mean`, `shall have
|
|
4759
|
+
* the meaning(s)`, `refers to`). Legal documents reserve
|
|
4760
|
+
* this construction for defined terms; the contents are
|
|
4761
|
+
* not personal names even when individual tokens collide
|
|
4762
|
+
* with the name corpus.
|
|
4763
|
+
*
|
|
4764
|
+
* Plain quotations like `"John Unknown" said ...` do NOT
|
|
4765
|
+
* count: there is no definitional cue, so the trailing
|
|
4766
|
+
* surname extension is still allowed to absorb `Unknown`.
|
|
4767
|
+
*/
|
|
4768
|
+
const OPENING_QUOTES = new Set([
|
|
4769
|
+
"\"",
|
|
4770
|
+
"'",
|
|
4771
|
+
"“",
|
|
4772
|
+
"„",
|
|
4773
|
+
"‟",
|
|
4774
|
+
"‘",
|
|
4775
|
+
"‛",
|
|
4776
|
+
"«"
|
|
4777
|
+
]);
|
|
4778
|
+
const CLOSING_QUOTES = new Set([
|
|
4779
|
+
"\"",
|
|
4780
|
+
"'",
|
|
4781
|
+
"”",
|
|
4782
|
+
"’",
|
|
4783
|
+
"»",
|
|
4784
|
+
"“"
|
|
4785
|
+
]);
|
|
4786
|
+
const DEFINED_TERM_CUE_RE = /^[\s,]*(?:means?|shall\s+means?|shall\s+have\s+the\s+meanings?|refers?\s+to|has\s+the\s+meanings?|is\s+defined)\b/iu;
|
|
4787
|
+
const DEFINED_TERM_LOOKAHEAD = 120;
|
|
4788
|
+
const DEFINED_TERM_LOOKBEHIND = 80;
|
|
4789
|
+
const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
|
|
4790
|
+
const isLetter = (ch) => ch !== void 0 && /^\p{L}$/u.test(ch);
|
|
4791
|
+
const isApostropheInsideWord = (text, index) => isLetter(text[index - 1]) && isLetter(text[index + 1]);
|
|
4792
|
+
const isQuoteBoundary = (text, index) => {
|
|
4793
|
+
const ch = text[index];
|
|
4794
|
+
if (ch !== "'" && ch !== "’") return true;
|
|
4795
|
+
return !isApostropheInsideWord(text, index);
|
|
4796
|
+
};
|
|
4797
|
+
const findDefinedTermQuoteContent = (text, start) => {
|
|
4798
|
+
const min = Math.max(0, start - DEFINED_TERM_LOOKBEHIND);
|
|
4799
|
+
let quoteStart = -1;
|
|
4800
|
+
for (let i = start - 1; i >= min; i--) {
|
|
4801
|
+
const ch = text[i];
|
|
4802
|
+
if (ch === "\n") break;
|
|
4803
|
+
if (ch && OPENING_QUOTES.has(ch) && isQuoteBoundary(text, i)) {
|
|
4804
|
+
quoteStart = i;
|
|
4805
|
+
break;
|
|
4806
|
+
}
|
|
4807
|
+
if (ch && CLOSING_QUOTES.has(ch) && isQuoteBoundary(text, i)) break;
|
|
4808
|
+
}
|
|
4809
|
+
if (quoteStart === -1) return null;
|
|
4810
|
+
const max = Math.min(text.length, quoteStart + 1 + DEFINED_TERM_LOOKAHEAD);
|
|
4811
|
+
for (let i = start; i < max; i++) {
|
|
4812
|
+
const ch = text[i];
|
|
4813
|
+
if (!ch || !CLOSING_QUOTES.has(ch) || !isQuoteBoundary(text, i)) continue;
|
|
4814
|
+
const after = text.slice(i + 1, max);
|
|
4815
|
+
if (!DEFINED_TERM_CUE_RE.test(after)) return null;
|
|
4816
|
+
return {
|
|
4817
|
+
content: text.slice(quoteStart + 1, i),
|
|
4818
|
+
afterClosingQuote: after
|
|
4819
|
+
};
|
|
4820
|
+
}
|
|
4821
|
+
return null;
|
|
4822
|
+
};
|
|
4823
|
+
const FIRST_WORD_RE = /^\p{L}+/u;
|
|
4824
|
+
const WORD_RE = /\p{L}+/gu;
|
|
4825
|
+
const startsWithKnownFirstName = (quoteContent, ctx) => {
|
|
4826
|
+
const firstWord = FIRST_WORD_RE.exec(quoteContent.trim())?.[0];
|
|
4827
|
+
if (!firstWord) return false;
|
|
4828
|
+
return new Set(getNameCorpusFirstNames(ctx).map((name) => name.toLowerCase())).has(firstWord.toLowerCase());
|
|
4829
|
+
};
|
|
4830
|
+
const hasPersonRoleDefinition = (afterClosingQuote, ctx) => {
|
|
4831
|
+
const roleWords = afterClosingQuote.replace(DEFINED_TERM_CUE_RE, "").match(WORD_RE)?.slice(0, 8) ?? [];
|
|
4832
|
+
if (roleWords.length === 0) return false;
|
|
4833
|
+
const genericRoles = ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
4834
|
+
return roleWords.some((word) => genericRoles.has(word.toLowerCase()));
|
|
4835
|
+
};
|
|
4836
|
+
const isSuppressibleDefinedTermQuote = (text, start, ctx) => {
|
|
4837
|
+
const definedTermQuote = findDefinedTermQuoteContent(text, start);
|
|
4838
|
+
if (definedTermQuote === null) return false;
|
|
4839
|
+
const words = definedTermQuote.content.match(WORD_RE) ?? [];
|
|
4840
|
+
if (words.length >= 2 && startsWithKnownFirstName(definedTermQuote.content, ctx) && hasPersonRoleDefinition(definedTermQuote.afterClosingQuote, ctx)) return false;
|
|
4841
|
+
return words.length >= 2;
|
|
4842
|
+
};
|
|
4452
4843
|
const extendPersonName = (text, start, end, ctx) => {
|
|
4453
4844
|
let newEnd = end;
|
|
4454
4845
|
let pos = newEnd;
|
|
@@ -4459,7 +4850,7 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
4459
4850
|
if (!UPPER_START_RE.test(char)) break;
|
|
4460
4851
|
let wordEnd = wordStart;
|
|
4461
4852
|
while (wordEnd < text.length && !/\s/.test(text[wordEnd] ?? "")) wordEnd++;
|
|
4462
|
-
const stripped = text.slice(wordStart, wordEnd).replace(/[
|
|
4853
|
+
const stripped = text.slice(wordStart, wordEnd).replace(/[,;.”"’'“»]+$/, "");
|
|
4463
4854
|
if (stripped.length < 2) break;
|
|
4464
4855
|
const lower = stripped.toLowerCase();
|
|
4465
4856
|
if (getStopwords(ctx).has(lower) || getPersonStopwords(ctx).has(lower)) break;
|
|
@@ -4482,6 +4873,15 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
4482
4873
|
* cluster occasionally absorbs.
|
|
4483
4874
|
*/
|
|
4484
4875
|
const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
|
|
4876
|
+
const POSTAL_ADJACENT = `\\p{L}\\p{N}_${DASH_INNER}`;
|
|
4877
|
+
const POSTAL_CODE_RE = new RegExp(`(?<![${POSTAL_ADJACENT}])(?:\\d{3}\\s\\d{2}|\\d{2}${DASH}\\d{3}|\\d{5}${DASH}\\d{3}|\\d{5}${DASH}\\d{4})(?![${POSTAL_ADJACENT}])`, "gu");
|
|
4878
|
+
const BR_CEP_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{3}$`, "u");
|
|
4879
|
+
const US_ZIP_PLUS_FOUR_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{4}$`, "u");
|
|
4880
|
+
const US_STATE_ABBREV_BEFORE_ZIP_RE = new RegExp(`(?:^|[^A-Za-z0-9])(A[KLRZ]|C[AOT]|D[CE]|F[LM]|G[AU]|HI|I[ADLN]|K[SY]|LA|M[ADEHINOPST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\\s*,?\\s*$`, "u");
|
|
4881
|
+
const US_ZIP_CONTEXT_WINDOW = 120;
|
|
4882
|
+
const US_CITY_ZIP_GAP_RE = /^[\s,]+$/u;
|
|
4883
|
+
const HOUSE_NUMBER_BEFORE_STREET_RE = /\b\d{1,6}(?:[-/]\d{1,6})?\s+(?:\p{Lu}\p{L}+[^\S\n\t]+){0,4}$/u;
|
|
4884
|
+
const HOUSE_NUMBER_AFTER_STREET_RE = /^[^\S\n\t]+\d{1,6}(?:[-/]\d{1,6})?\b/u;
|
|
4485
4885
|
let cachedBoundaryRe = null;
|
|
4486
4886
|
const loadBoundaryWords = async () => {
|
|
4487
4887
|
try {
|
|
@@ -4496,7 +4896,7 @@ let cachedBrCepContextPromise = null;
|
|
|
4496
4896
|
const loadBrCueWords = async () => {
|
|
4497
4897
|
const sources = await Promise.all([(async () => {
|
|
4498
4898
|
try {
|
|
4499
|
-
return (await import("./address-street-types.mjs")).default["pt-br"];
|
|
4899
|
+
return (await import("./address-street-types.mjs").then((n) => n.n)).default["pt-br"];
|
|
4500
4900
|
} catch {
|
|
4501
4901
|
return;
|
|
4502
4902
|
}
|
|
@@ -4540,6 +4940,48 @@ const hasBrCueNearby = (fullText, start, end, re) => {
|
|
|
4540
4940
|
const window = fullText.slice(windowStart, windowEnd);
|
|
4541
4941
|
return new RegExp(re.source, re.flags.replace("g", "")).test(window);
|
|
4542
4942
|
};
|
|
4943
|
+
const getUsStateSeedBeforeZip = (fullText, start) => {
|
|
4944
|
+
const stateWindowStart = Math.max(0, start - 24);
|
|
4945
|
+
const stateWindow = fullText.slice(stateWindowStart, start);
|
|
4946
|
+
const match = US_STATE_ABBREV_BEFORE_ZIP_RE.exec(stateWindow);
|
|
4947
|
+
const state = match?.[1];
|
|
4948
|
+
if (!match || !state) return null;
|
|
4949
|
+
const stateOffset = match[0].indexOf(state);
|
|
4950
|
+
const stateStart = stateWindowStart + match.index + stateOffset;
|
|
4951
|
+
return {
|
|
4952
|
+
type: "state",
|
|
4953
|
+
start: stateStart,
|
|
4954
|
+
end: stateStart + state.length,
|
|
4955
|
+
text: state
|
|
4956
|
+
};
|
|
4957
|
+
};
|
|
4958
|
+
const hasHouseNumberNearStreetWord = (fullText, seed) => {
|
|
4959
|
+
if (/\d/.test(seed.text)) return true;
|
|
4960
|
+
const before = fullText.slice(Math.max(0, seed.start - 50), seed.start);
|
|
4961
|
+
if (HOUSE_NUMBER_BEFORE_STREET_RE.test(before)) return true;
|
|
4962
|
+
const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
|
|
4963
|
+
return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
|
|
4964
|
+
};
|
|
4965
|
+
const getUsZipPlusFourContext = (fullText, start, seeds) => {
|
|
4966
|
+
const stateSeed = getUsStateSeedBeforeZip(fullText, start);
|
|
4967
|
+
if (stateSeed !== null) return {
|
|
4968
|
+
stateSeed,
|
|
4969
|
+
hasContext: true
|
|
4970
|
+
};
|
|
4971
|
+
return {
|
|
4972
|
+
stateSeed: null,
|
|
4973
|
+
hasContext: seeds.some((seed) => {
|
|
4974
|
+
if (Math.abs(seed.start - start) > US_ZIP_CONTEXT_WINDOW) return false;
|
|
4975
|
+
if (seed.type === "address-trigger") return true;
|
|
4976
|
+
if (seed.type === "city" && seed.end <= start) {
|
|
4977
|
+
const gap = fullText.slice(seed.end, start);
|
|
4978
|
+
return US_CITY_ZIP_GAP_RE.test(gap);
|
|
4979
|
+
}
|
|
4980
|
+
if (seed.type === "street-word") return hasHouseNumberNearStreetWord(fullText, seed);
|
|
4981
|
+
return false;
|
|
4982
|
+
})
|
|
4983
|
+
};
|
|
4984
|
+
};
|
|
4543
4985
|
/**
|
|
4544
4986
|
* Build regex for boundary words. Matches any
|
|
4545
4987
|
* boundary word preceded by a word boundary.
|
|
@@ -4564,7 +5006,7 @@ const getBoundaryRe = async () => {
|
|
|
4564
5006
|
const buildStreetTypePatterns = async () => {
|
|
4565
5007
|
let config = {};
|
|
4566
5008
|
try {
|
|
4567
|
-
config = (await import("./address-street-types.mjs")).default;
|
|
5009
|
+
config = (await import("./address-street-types.mjs").then((n) => n.n)).default;
|
|
4568
5010
|
} catch {
|
|
4569
5011
|
return [];
|
|
4570
5012
|
}
|
|
@@ -4609,13 +5051,22 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
4609
5051
|
text: e.text
|
|
4610
5052
|
});
|
|
4611
5053
|
}
|
|
4612
|
-
const postalRe =
|
|
5054
|
+
const postalRe = POSTAL_CODE_RE;
|
|
5055
|
+
postalRe.lastIndex = 0;
|
|
4613
5056
|
let postalMatch;
|
|
4614
5057
|
while ((postalMatch = postalRe.exec(fullText)) !== null) {
|
|
4615
5058
|
const start = postalMatch.index;
|
|
4616
5059
|
const end = start + postalMatch[0].length;
|
|
4617
5060
|
if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
|
|
4618
|
-
if (
|
|
5061
|
+
if (BR_CEP_SHAPE_RE.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
|
|
5062
|
+
if (US_ZIP_PLUS_FOUR_SHAPE_RE.test(postalMatch[0])) {
|
|
5063
|
+
const usContext = getUsZipPlusFourContext(fullText, start, seeds);
|
|
5064
|
+
if (!usContext.hasContext) continue;
|
|
5065
|
+
const stateSeed = usContext.stateSeed;
|
|
5066
|
+
if (stateSeed !== null) {
|
|
5067
|
+
if (!seeds.some((seed) => seed.start === stateSeed.start && seed.end === stateSeed.end)) seeds.push(stateSeed);
|
|
5068
|
+
}
|
|
5069
|
+
}
|
|
4619
5070
|
seeds.push({
|
|
4620
5071
|
type: "postal-code",
|
|
4621
5072
|
start,
|
|
@@ -4694,6 +5145,7 @@ const scoreCluster = (cluster) => {
|
|
|
4694
5145
|
let score = .5;
|
|
4695
5146
|
if (types.has("postal-code")) score += .15;
|
|
4696
5147
|
if (types.has("city")) score += .15;
|
|
5148
|
+
if (types.has("state")) score += .15;
|
|
4697
5149
|
if (types.has("street-word")) score += .15;
|
|
4698
5150
|
if (types.has("address-trigger")) score += .1;
|
|
4699
5151
|
return Math.min(score, .95);
|
|
@@ -4880,7 +5332,44 @@ const BARE_STOPWORDS = new Set([
|
|
|
4880
5332
|
"Splatnost",
|
|
4881
5333
|
"Variabilní",
|
|
4882
5334
|
"Konstantní",
|
|
4883
|
-
"Specifický"
|
|
5335
|
+
"Specifický",
|
|
5336
|
+
"Section",
|
|
5337
|
+
"Sections",
|
|
5338
|
+
"Article",
|
|
5339
|
+
"Articles",
|
|
5340
|
+
"Schedule",
|
|
5341
|
+
"Schedules",
|
|
5342
|
+
"Exhibit",
|
|
5343
|
+
"Exhibits",
|
|
5344
|
+
"Annex",
|
|
5345
|
+
"Annexes",
|
|
5346
|
+
"Appendix",
|
|
5347
|
+
"Appendices",
|
|
5348
|
+
"Clause",
|
|
5349
|
+
"Clauses",
|
|
5350
|
+
"Chapter",
|
|
5351
|
+
"Chapters",
|
|
5352
|
+
"Paragraph",
|
|
5353
|
+
"Paragraphs",
|
|
5354
|
+
"Subsection",
|
|
5355
|
+
"Subsections",
|
|
5356
|
+
"Form",
|
|
5357
|
+
"Page",
|
|
5358
|
+
"Pages",
|
|
5359
|
+
"Item",
|
|
5360
|
+
"Items",
|
|
5361
|
+
"Note",
|
|
5362
|
+
"Notes",
|
|
5363
|
+
"Rule",
|
|
5364
|
+
"Rules",
|
|
5365
|
+
"Attachment",
|
|
5366
|
+
"Attachments",
|
|
5367
|
+
"Volume",
|
|
5368
|
+
"Volumes",
|
|
5369
|
+
"Book",
|
|
5370
|
+
"Books",
|
|
5371
|
+
"Part",
|
|
5372
|
+
"Parts"
|
|
4884
5373
|
]);
|
|
4885
5374
|
const NEAR_MISS_BAND = .15;
|
|
4886
5375
|
const BOOST_PER_NEIGHBOUR = .05;
|
|
@@ -4953,21 +5442,23 @@ const initPrepositions = () => {
|
|
|
4953
5442
|
};
|
|
4954
5443
|
const getAddressPreps = () => _addressPreps ?? /* @__PURE__ */ new Set();
|
|
4955
5444
|
const getTemporalPreps = () => _temporalPreps ?? /* @__PURE__ */ new Set();
|
|
4956
|
-
|
|
5445
|
+
const buildStreetAbbrevs = (data) => {
|
|
5446
|
+
const abbrevs = /* @__PURE__ */ new Set();
|
|
5447
|
+
for (const [key, words] of Object.entries(data)) {
|
|
5448
|
+
if (key.startsWith("_")) continue;
|
|
5449
|
+
if (!Array.isArray(words)) continue;
|
|
5450
|
+
for (const word of words) if (typeof word === "string" && word.includes(".")) abbrevs.add(word.toLowerCase());
|
|
5451
|
+
}
|
|
5452
|
+
return abbrevs;
|
|
5453
|
+
};
|
|
5454
|
+
let _streetAbbrevs = buildStreetAbbrevs(address_street_types_default);
|
|
4957
5455
|
let _streetAbbrevsPromise = null;
|
|
4958
5456
|
const loadStreetAbbrevs = async () => {
|
|
4959
5457
|
try {
|
|
4960
|
-
const mod = await import("./address-street-types.mjs");
|
|
4961
|
-
|
|
4962
|
-
const abbrevs = /* @__PURE__ */ new Set();
|
|
4963
|
-
for (const [key, words] of Object.entries(data)) {
|
|
4964
|
-
if (key.startsWith("_")) continue;
|
|
4965
|
-
if (!Array.isArray(words)) continue;
|
|
4966
|
-
for (const w of words) if (w.includes(".")) abbrevs.add(w.toLowerCase());
|
|
4967
|
-
}
|
|
4968
|
-
_streetAbbrevs = abbrevs;
|
|
5458
|
+
const mod = await import("./address-street-types.mjs").then((n) => n.n);
|
|
5459
|
+
_streetAbbrevs = buildStreetAbbrevs(mod.default ?? mod);
|
|
4969
5460
|
} catch {
|
|
4970
|
-
_streetAbbrevs
|
|
5461
|
+
_streetAbbrevs ??= /* @__PURE__ */ new Set();
|
|
4971
5462
|
}
|
|
4972
5463
|
};
|
|
4973
5464
|
/** Ensure street abbreviation data is loaded. */
|
|
@@ -5079,7 +5570,7 @@ const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
|
5079
5570
|
}
|
|
5080
5571
|
return results;
|
|
5081
5572
|
};
|
|
5082
|
-
const ORPHAN_STREET_RE =
|
|
5573
|
+
const ORPHAN_STREET_RE = /^[^\S\n]*(\p{Lu}[\p{Ll}\p{Lu}]+(?:[^\S\n]+[\p{Lu}\p{Ll}][\p{Ll}]+)*[^\S\n]+\d{2,4}[a-zA-Z]?)[^\S\n]*$/gmu;
|
|
5083
5574
|
/**
|
|
5084
5575
|
* In the header zone (top 15%), find standalone lines
|
|
5085
5576
|
* matching "[Uppercase word(s)] [number]" that sit
|
|
@@ -5457,6 +5948,7 @@ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-lis
|
|
|
5457
5948
|
* of `\s` to avoid merging entities across newlines.
|
|
5458
5949
|
*/
|
|
5459
5950
|
const GAP_PATTERN = /^[ \t,\-]+$/;
|
|
5951
|
+
const isLegalFormOrganization = (entity) => entity.label === "organization" && entity.source === DETECTION_SOURCES.LEGAL_FORM;
|
|
5460
5952
|
/**
|
|
5461
5953
|
* Build a set of word boundary offsets for the full
|
|
5462
5954
|
* text using `Intl.Segmenter`. Returns a sorted array
|
|
@@ -5596,7 +6088,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
5596
6088
|
break;
|
|
5597
6089
|
}
|
|
5598
6090
|
}
|
|
5599
|
-
if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
6091
|
+
if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev) && isLegalFormOrganization(entity) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
5600
6092
|
prev.end = entity.end;
|
|
5601
6093
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
5602
6094
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -5996,6 +6488,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
5996
6488
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
5997
6489
|
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
5998
6490
|
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
6491
|
+
const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
|
|
6492
|
+
const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
|
|
5999
6493
|
const shouldReplace = (a, b) => {
|
|
6000
6494
|
const aLen = a.end - a.start;
|
|
6001
6495
|
const bLen = b.end - b.start;
|
|
@@ -6011,6 +6505,31 @@ const shouldReplace = (a, b) => {
|
|
|
6011
6505
|
/** Labels where colons are structurally significant. */
|
|
6012
6506
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
6013
6507
|
/**
|
|
6508
|
+
* Labels whose entities should have a trailing sentence
|
|
6509
|
+
* `.` stripped during sanitisation. Restricted to
|
|
6510
|
+
* proper-noun-style labels where a final period is
|
|
6511
|
+
* almost always the sentence terminator that ran into
|
|
6512
|
+
* the capture, not a structural part of the value.
|
|
6513
|
+
* Numeric labels (`date`, `date of birth`, `phone
|
|
6514
|
+
* number`, `monetary amount`, `time`) and `person`
|
|
6515
|
+
* stay out — German writes `21. März`, post-nominal
|
|
6516
|
+
* degrees write `M.Sc.`, times write `5:00 p.m.`, and
|
|
6517
|
+
* stripping the dot would corrupt those spans.
|
|
6518
|
+
*/
|
|
6519
|
+
const PERIOD_STRIPPED_LABELS = new Set([
|
|
6520
|
+
"organization",
|
|
6521
|
+
"location",
|
|
6522
|
+
"address"
|
|
6523
|
+
]);
|
|
6524
|
+
const ADDRESS_FINAL_TOKEN_RE = /(?:^|[\s,])([\p{L}\p{M}.]+\.)$/u;
|
|
6525
|
+
const LOCATION_FINAL_DOTTED_ABBREV_RE = /(?:^|[\s,])(?:\p{Lu}\.){2,}$/u;
|
|
6526
|
+
const hasKnownAddressFinalAbbrev = (text) => {
|
|
6527
|
+
const finalToken = ADDRESS_FINAL_TOKEN_RE.exec(text)?.[1];
|
|
6528
|
+
if (!finalToken) return false;
|
|
6529
|
+
return getStreetAbbrevs().has(finalToken.toLowerCase());
|
|
6530
|
+
};
|
|
6531
|
+
const hasLocationFinalAbbrev = (text) => LOCATION_FINAL_DOTTED_ABBREV_RE.test(text);
|
|
6532
|
+
/**
|
|
6014
6533
|
* Labels whose detectors emit precise, evidence-backed spans. When
|
|
6015
6534
|
* one of these fires at the exact same offsets as a fuzzier
|
|
6016
6535
|
* `address` hit (city dictionary lookup, address-seed cluster), the
|
|
@@ -6083,16 +6602,63 @@ const resolveSameSpanLabelConflicts = (entities) => {
|
|
|
6083
6602
|
if (dropped.size === 0) return entities;
|
|
6084
6603
|
return entities.filter((e) => !dropped.has(e));
|
|
6085
6604
|
};
|
|
6605
|
+
/**
|
|
6606
|
+
* Trailing typographic punctuation that detectors
|
|
6607
|
+
* occasionally swallow when a capture runs to the end
|
|
6608
|
+
* of a sentence or quoted phrase. Stripped from every
|
|
6609
|
+
* non-literal, non-locked entity. Curated dictionary and
|
|
6610
|
+
* gazetteer entries with punctuation that is clearly part of
|
|
6611
|
+
* the literal (`Hello bank!`, `"Juez y parte"`) keep their
|
|
6612
|
+
* own boundaries. Generated/extended spans from the same
|
|
6613
|
+
* sources still pass through cleanup so dangling punctuation
|
|
6614
|
+
* does not become part of the redaction
|
|
6615
|
+
* (e.g. `Bond Hedge Documentation"` →
|
|
6616
|
+
* `Bond Hedge Documentation`).
|
|
6617
|
+
*
|
|
6618
|
+
* `)` is deliberately omitted — monetary amounts are
|
|
6619
|
+
* extended to include trailing "(slovy ...)" / "(in
|
|
6620
|
+
* words ...)" parentheticals where the closing paren
|
|
6621
|
+
* is structural, and stripping it would leave the open
|
|
6622
|
+
* paren dangling. `.` is also omitted because the
|
|
6623
|
+
* trailing-period rule below has label-aware handling
|
|
6624
|
+
* (legal-form abbreviations keep their dot).
|
|
6625
|
+
*/
|
|
6626
|
+
const TRAILING_PUNCT_CLASS = `["“”‘’'»!?]`;
|
|
6627
|
+
/**
|
|
6628
|
+
* Leading typographic punctuation that detectors
|
|
6629
|
+
* occasionally swallow when a capture starts at an
|
|
6630
|
+
* opening quote. `(` is deliberately omitted — it is
|
|
6631
|
+
* almost always the opening of a structural
|
|
6632
|
+
* parenthetical (registration number group, monetary
|
|
6633
|
+
* "(slovy ...)" extension) that the detector
|
|
6634
|
+
* intentionally captured.
|
|
6635
|
+
*/
|
|
6636
|
+
const LEADING_PUNCT_CLASS = `["“”‘’'«¿¡]`;
|
|
6637
|
+
const STRIP_BY_LABEL = {
|
|
6638
|
+
colon: /[\s,;]+/,
|
|
6639
|
+
default: /[\s:,;]+/
|
|
6640
|
+
};
|
|
6641
|
+
const LEADING_TRIM_BY_LABEL = {
|
|
6642
|
+
colon: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.colon.source}|${LEADING_PUNCT_CLASS})+`),
|
|
6643
|
+
default: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.default.source}|${LEADING_PUNCT_CLASS})+`)
|
|
6644
|
+
};
|
|
6645
|
+
const TRAILING_TRIM_BY_LABEL = {
|
|
6646
|
+
colon: new RegExp(`(?:${STRIP_BY_LABEL.colon.source}|${TRAILING_PUNCT_CLASS})+$`),
|
|
6647
|
+
default: new RegExp(`(?:${STRIP_BY_LABEL.default.source}|${TRAILING_PUNCT_CLASS})+$`)
|
|
6648
|
+
};
|
|
6086
6649
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
6087
6650
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
6088
|
-
if (hasLockedBoundary(e)) return [e];
|
|
6089
|
-
const
|
|
6090
|
-
const
|
|
6651
|
+
if (hasLockedBoundary(e) || hasCuratedLiteralBoundary(e)) return [e];
|
|
6652
|
+
const stripKind = COLON_LABELS.has(e.label) ? "colon" : "default";
|
|
6653
|
+
const leadRe = LEADING_TRIM_BY_LABEL[stripKind];
|
|
6654
|
+
const trailRe = TRAILING_TRIM_BY_LABEL[stripKind];
|
|
6655
|
+
const leadTrimmed = e.text.replace(leadRe, "");
|
|
6091
6656
|
const lead = e.text.length - leadTrimmed.length;
|
|
6092
|
-
let cleaned = leadTrimmed.replace(
|
|
6093
|
-
if (e.label
|
|
6094
|
-
if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6657
|
+
let cleaned = leadTrimmed.replace(trailRe, "");
|
|
6658
|
+
if (PERIOD_STRIPPED_LABELS.has(e.label) && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
|
|
6659
|
+
if (!(getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix)) || e.label === "address" && hasKnownAddressFinalAbbrev(cleaned) || e.label === "location" && hasLocationFinalAbbrev(cleaned))) cleaned = cleaned.slice(0, -1).trimEnd();
|
|
6095
6660
|
}
|
|
6661
|
+
cleaned = cleaned.replace(trailRe, "");
|
|
6096
6662
|
if (cleaned.length === 0) return [];
|
|
6097
6663
|
if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
|
|
6098
6664
|
const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
|
|
@@ -6149,7 +6715,7 @@ let amountWordsLoaded = false;
|
|
|
6149
6715
|
const getAmountWordsRe = async () => {
|
|
6150
6716
|
if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
|
|
6151
6717
|
try {
|
|
6152
|
-
const alt = (await import("./amount-words.mjs")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
6718
|
+
const alt = (await import("./amount-words.mjs").then((n) => n.n)).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
6153
6719
|
amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
|
|
6154
6720
|
} catch {
|
|
6155
6721
|
amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
|