@stll/anonymize-wasm 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/wasm.mjs CHANGED
@@ -1,4 +1,6 @@
1
1
  import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
2
+ import { t as amount_words_default } from "./amount-words.mjs";
3
+ import { t as address_street_types_default } from "./address-street-types.mjs";
2
4
  import { TextSearch } from "@stll/text-search-wasm";
3
5
  import { at, be, bg, br, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
4
6
  import { toRegex } from "@stll/stdnum/patterns";
@@ -885,6 +887,7 @@ const getSentenceVerbIndicatorsSync = () => sentenceVerbIndicatorsCache ?? SENTE
885
887
  const UPPER = "A-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽÄÖÜÀÂÆÇÈÊËÎÏÔÙÛŸÑĄĆĘŁŃŚŹŻ\\u0130";
886
888
  const LOWER = "a-záčďéěíňóřšťúůýžäöüßàâæçèêëîïôùûÿñąćęłńśźż\\u0131";
887
889
  const CAP_WORD = `(?:[${UPPER}]{2,}|[${UPPER}][${LOWER}${UPPER}]+)`;
890
+ const SINGLE_CAP = `[${UPPER}](?![${LOWER}${UPPER}])`;
888
891
  const ALLCAP_WORD = `[${UPPER}]{2,}`;
889
892
  const ROMAN_NUMERAL_RE = /^(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/;
890
893
  let legalRoleHeadsCache = null;
@@ -912,11 +915,22 @@ const warmLegalRoleHeads = async () => {
912
915
  loadLegalRoleHeads(),
913
916
  loadAllLegalSuffixes(),
914
917
  loadSentenceVerbIndicators(),
915
- loadClauseNounHeads()
918
+ loadClauseNounHeads(),
919
+ loadConnectorProseHeads(),
920
+ loadStructuralSingleCapPrefixes()
916
921
  ]);
917
922
  };
918
923
  let allLegalSuffixesCache = null;
919
924
  let allLegalSuffixesPromise = null;
925
+ let normalizedLegalBoundarySuffixesCache = null;
926
+ let normalizedInNameLegalFormWordsCache = null;
927
+ const normalizeLegalSuffixToken = (suffix) => suffix.replace(/[.,\s]/g, "");
928
+ const isBoundaryLegalSuffixForm = (form) => {
929
+ const normalized = normalizeLegalSuffixToken(form);
930
+ if (normalized.length === 0) return false;
931
+ if (LEGAL_SUFFIXES.includes(form)) return true;
932
+ return /[.]/u.test(form) || normalized === normalized.toUpperCase();
933
+ };
920
934
  const loadAllLegalSuffixes = async () => {
921
935
  if (allLegalSuffixesCache) return allLegalSuffixesCache;
922
936
  if (allLegalSuffixesPromise) return allLegalSuffixesPromise;
@@ -941,11 +955,15 @@ const loadAllLegalSuffixes = async () => {
941
955
  }
942
956
  out.sort((a, b) => b.length - a.length);
943
957
  allLegalSuffixesCache = out;
958
+ normalizedLegalBoundarySuffixesCache = new Set(out.filter(isBoundaryLegalSuffixForm).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
959
+ normalizedInNameLegalFormWordsCache = new Set(out.filter((form) => !isBoundaryLegalSuffixForm(form) && !/\s/u.test(form)).map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
944
960
  return out;
945
961
  })();
946
962
  return allLegalSuffixesPromise;
947
963
  };
948
964
  const getAllLegalSuffixesSync = () => allLegalSuffixesCache ?? LEGAL_SUFFIXES;
965
+ const getNormalizedLegalBoundarySuffixesSync = () => normalizedLegalBoundarySuffixesCache ?? new Set(LEGAL_SUFFIXES.map(normalizeLegalSuffixToken).filter((suffix) => suffix.length > 0));
966
+ const getNormalizedInNameLegalFormWordsSync = () => normalizedInNameLegalFormWordsCache ?? /* @__PURE__ */ new Set();
949
967
  /**
950
968
  * Sync accessor for the full legal-form vocabulary
951
969
  * (`data/legal-forms.json` plus `LEGAL_SUFFIXES`,
@@ -985,18 +1003,72 @@ const loadClauseNounHeads = async () => {
985
1003
  return clauseNounHeadsPromise;
986
1004
  };
987
1005
  const getClauseNounHeadsSync = () => clauseNounHeadsCache ?? CLAUSE_NOUN_HEADS_SEED;
1006
+ let connectorProseHeadsCache = null;
1007
+ let connectorProseHeadsPromise = null;
1008
+ const loadConnectorProseHeads = async () => {
1009
+ if (connectorProseHeadsCache) return connectorProseHeadsCache;
1010
+ if (connectorProseHeadsPromise) return connectorProseHeadsPromise;
1011
+ connectorProseHeadsPromise = (async () => {
1012
+ let data = {};
1013
+ try {
1014
+ const mod = await import("./generic-roles.mjs");
1015
+ data = mod.default ?? mod;
1016
+ } catch (err) {
1017
+ console.warn("[anonymize] legal-forms: failed to load generic-roles.json:", err);
1018
+ }
1019
+ const all = /* @__PURE__ */ new Set();
1020
+ if (Array.isArray(data.roles)) {
1021
+ for (const role of data.roles) if (typeof role === "string" && role.length > 0) all.add(role.toLowerCase());
1022
+ }
1023
+ connectorProseHeadsCache = all;
1024
+ return all;
1025
+ })();
1026
+ return connectorProseHeadsPromise;
1027
+ };
1028
+ const getConnectorProseHeadsSync = () => connectorProseHeadsCache ?? /* @__PURE__ */ new Set();
1029
+ let structuralSingleCapPrefixesCache = null;
1030
+ let structuralSingleCapPrefixesPromise = null;
1031
+ const loadStructuralSingleCapPrefixes = async () => {
1032
+ if (structuralSingleCapPrefixesCache) return structuralSingleCapPrefixesCache;
1033
+ if (structuralSingleCapPrefixesPromise) return structuralSingleCapPrefixesPromise;
1034
+ structuralSingleCapPrefixesPromise = (async () => {
1035
+ let data = {};
1036
+ try {
1037
+ const mod = await import("./structural-single-cap-prefixes.mjs");
1038
+ data = mod.default ?? mod;
1039
+ } catch (err) {
1040
+ console.warn("[anonymize] legal-forms: failed to load structural-single-cap-prefixes.json:", err);
1041
+ }
1042
+ const all = /* @__PURE__ */ new Set();
1043
+ for (const [key, value] of Object.entries(data)) {
1044
+ if (key.startsWith("_")) continue;
1045
+ if (!Array.isArray(value)) continue;
1046
+ for (const prefix of value) {
1047
+ if (typeof prefix !== "string" || prefix.length === 0) continue;
1048
+ all.add(prefix.toLowerCase());
1049
+ }
1050
+ }
1051
+ structuralSingleCapPrefixesCache = all;
1052
+ return all;
1053
+ })();
1054
+ return structuralSingleCapPrefixesPromise;
1055
+ };
1056
+ const getStructuralSingleCapPrefixesSync = () => structuralSingleCapPrefixesCache ?? /* @__PURE__ */ new Set();
988
1057
  const escapeForRegex = (form) => form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s+").replace(/\\\./g, "\\.[^\\S\\n]?");
989
1058
  const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.includes(" ");
1059
+ const buildDottedAbbreviationAlternation = (forms) => [...new Set(forms.filter((form) => /^[\p{Lu}][\p{L}\p{M}]{0,5}\.$/u.test(form)).map((form) => form.slice(0, -1)).filter((form) => form.length > 0))].toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
990
1060
  const buildPatternString = (forms) => {
991
1061
  if (forms.length === 0) return null;
992
1062
  const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
993
1063
  const HSPACE = "[^\\S\\n]";
994
1064
  const LOWER_CONNECTOR = `${HSPACE}+(?:a|and|und|et|e|y|i)${HSPACE}+(?=[${LOWER}])`;
995
1065
  const SIMPLE_SEP = `(?:${HSPACE}|[&,.${DASH_INNER}]){1,4}`;
996
- const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|\\d{1,4})`;
1066
+ const CAP_OR_NUM_WORD = `(?:${CAP_WORD}|${SINGLE_CAP}|\\d{1,4})`;
997
1067
  const LOWER_WORD = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${LOWER}][${LOWER}${UPPER}]+)`;
998
1068
  const ANY_WORD_TAIL = `(?:(?!(?:and|und|et)(?![${UPPER}${LOWER}]))[${UPPER}${LOWER}][${LOWER}${UPPER}]+|[${UPPER}]{2,3}|\\d{1,4})`;
999
- return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}`})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1069
+ const dottedAbbreviationAlt = buildDottedAbbreviationAlternation(forms);
1070
+ const dottedAbbreviationTail = dottedAbbreviationAlt.length > 0 ? `(?:${SIMPLE_SEP}(?:${dottedAbbreviationAlt})\\.)?` : "";
1071
+ return `${`(?:${`(?:${CAP_WORD})(?:${SIMPLE_SEP}(?:${CAP_OR_NUM_WORD})){0,10}` + dottedAbbreviationTail})(?:${`${SIMPLE_SEP}(?:${LOWER_WORD})(?:(?:${LOWER_CONNECTOR}|${SIMPLE_SEP})(?:${ANY_WORD_TAIL})){0,10}`})?`}(?:\\s+|,\\s*)(?:${alt})(?![${LOWER}])`;
1000
1072
  };
1001
1073
  /**
1002
1074
  * Build legal form regex pattern strings.
@@ -1029,6 +1101,7 @@ const buildLegalFormPatterns = async () => {
1029
1101
  const allcapPrefix = `(?:${ALLCAP_WORD})(?:[ \\t&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
1030
1102
  const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
1031
1103
  patterns.push(`${allcapPrefix}(?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${LOWER}])`);
1104
+ patterns.push(`(?:^|(?<=[^${UPPER}${LOWER}\\p{N}]))[${UPPER}](?:[ \\t]+|,[ \\t]*)(?:${allcapAlt})(?![${UPPER}${LOWER}\\p{N}])`);
1032
1105
  return patterns;
1033
1106
  };
1034
1107
  const CONNECTOR_RE = /^(?:a|and|und|et|e|y|i|&)$/i;
@@ -1038,6 +1111,12 @@ const COMPANY_SUFFIX_WORDS_RE = /^(?:Company|Co|Bank|Brothers|Bros|Sons|Group|Ho
1038
1111
  const IN_NAME_PREPOSITION_RE = /^(?:of|the)$/i;
1039
1112
  const ENTITY_HEAD_WORD_RE = /^[\p{L}\p{M}&]+/u;
1040
1113
  const LEADING_CLAUSE_RE = /(?:^|\s)(?:by\s+and\s+between|is\s+between)\s+/giu;
1114
+ const BARE_SINGLE_CAP_LEGAL_FORM_RE = new RegExp(`^[${UPPER}](?:[ \\t]+|,[ \\t]*)`, "u");
1115
+ const STRUCTURAL_SINGLE_CAP_RE = new RegExp(`^([\\p{L}\\p{M}]+)[ \\t]+[${UPPER}](?:[.${DASH_INNER}]?\\d{1,3})?(?:[ \\t]+|,[ \\t]*)`, "u");
1116
+ const isStructuralSingleCapMatch = (text) => {
1117
+ const first = STRUCTURAL_SINGLE_CAP_RE.exec(text)?.[1];
1118
+ return first !== void 0 && getStructuralSingleCapPrefixesSync().has(first.toLowerCase());
1119
+ };
1041
1120
  /**
1042
1121
  * Find the word ending just before `pos` in `text`,
1043
1122
  * skipping any whitespace (not newlines).
@@ -1062,25 +1141,145 @@ const findWordBefore = (text, pos) => {
1062
1141
  start: wordStart
1063
1142
  };
1064
1143
  };
1144
+ const hasSingleCapPrefixBefore = (fullText, matchStart) => {
1145
+ const prev = findWordBefore(fullText, matchStart);
1146
+ return prev !== null && prev.word.length === 1 && UPPER_LETTER_RE.test(prev.word);
1147
+ };
1148
+ const isBareSingleCapStructuralInnerMatch = (fullText, matchStart, text) => {
1149
+ if (!BARE_SINGLE_CAP_LEGAL_FORM_RE.test(text)) return false;
1150
+ const prev = findWordBefore(fullText, matchStart);
1151
+ return prev !== null && getStructuralSingleCapPrefixesSync().has(prev.word.toLowerCase());
1152
+ };
1153
+ const trimEmbeddedLegalFormListPrefix = (entityStart, entityText) => {
1154
+ let cut = -1;
1155
+ for (const suffix of getAllLegalSuffixesSync()) {
1156
+ const suffixClean = suffix.replace(/[.,\s]/g, "");
1157
+ if (suffixClean.length > 0 && ROMAN_NUMERAL_RE.test(suffixClean)) continue;
1158
+ let fromIndex = 0;
1159
+ while (fromIndex < entityText.length) {
1160
+ const suffixStart = entityText.indexOf(suffix, fromIndex);
1161
+ if (suffixStart === -1) break;
1162
+ fromIndex = suffixStart + suffix.length;
1163
+ const suffixEnd = suffixStart + suffix.length;
1164
+ if (suffixEnd >= entityText.length - 1) continue;
1165
+ const afterSuffix = entityText.slice(suffixEnd);
1166
+ const boundary = /^,\s+(?=\p{Lu})/u.exec(afterSuffix);
1167
+ if (boundary === null) continue;
1168
+ const nextStart = suffixEnd + boundary[0].length;
1169
+ const remainder = entityText.slice(nextStart);
1170
+ if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
1171
+ cut = Math.max(cut, nextStart);
1172
+ }
1173
+ }
1174
+ if (cut <= 0) return {
1175
+ entityStart,
1176
+ entityText
1177
+ };
1178
+ return {
1179
+ entityStart: entityStart + cut,
1180
+ entityText: entityText.slice(cut)
1181
+ };
1182
+ };
1183
+ const hasMiddleInitialBefore = (fullText, pos) => {
1184
+ const previousWord = findWordBefore(fullText, pos);
1185
+ if (!previousWord) return false;
1186
+ let scan = previousWord.start - 1;
1187
+ while (scan >= 0 && (fullText[scan] === " " || fullText[scan] === " ")) scan--;
1188
+ return scan >= 1 && fullText[scan] === "." && UPPER_LETTER_RE.test(fullText[scan - 1] ?? "");
1189
+ };
1065
1190
  /**
1066
1191
  * Count consecutive uppercase-starting words immediately
1067
1192
  * before `pos`. Stops at the first non-upper word or at
1068
- * text/line start. Used to disambiguate "<First> <Last>
1069
- * and <ORG>" from "<Multi-word Org> and <Continuation>".
1070
- */
1071
- const countUpperWordsBefore = (fullText, pos) => {
1193
+ * text/line start. Used to disambiguate sentence prose
1194
+ * ("<First> <Last> and <ORG>", "<Defined-Term> and
1195
+ * <ORG>") from multi-word organisation names that span
1196
+ * an "and" connector ("UniCredit Bank Czech Republic and
1197
+ * Slovakia, a.s.").
1198
+ *
1199
+ * When `crossInNamePreps` is true, the walk also steps
1200
+ * over in-name lowercase prepositions ("of", "the") as
1201
+ * long as they sit between two upper words. This lets
1202
+ * the suffix-mode "and"-crossing logic see through
1203
+ * "<Trust ← and ← America ← of ← Bank>" and emit one
1204
+ * full organisation span.
1205
+ */
1206
+ const countUpperWordsBefore = (fullText, pos, crossInNamePreps = false) => {
1072
1207
  let count = 0;
1073
1208
  let scan = pos;
1074
1209
  while (scan > 0) {
1075
1210
  const found = findWordBefore(fullText, scan);
1076
- if (!found) break;
1077
- if (!UPPER_LETTER_RE.test(found.word)) break;
1078
- count++;
1079
- scan = found.start;
1211
+ if (found) {
1212
+ if (UPPER_LETTER_RE.test(found.word)) {
1213
+ count++;
1214
+ scan = found.start;
1215
+ continue;
1216
+ }
1217
+ if (crossInNamePreps && IN_NAME_PREPOSITION_RE.test(found.word)) {
1218
+ const prev = findWordBefore(fullText, found.start);
1219
+ if (!prev) break;
1220
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1221
+ scan = found.start;
1222
+ continue;
1223
+ }
1224
+ break;
1225
+ }
1226
+ let p = scan - 1;
1227
+ while (p >= 0 && (fullText[p] === " " || fullText[p] === " ")) p--;
1228
+ if (p >= 1 && fullText[p] === "." && UPPER_LETTER_RE.test(fullText[p - 1] ?? "")) {
1229
+ count++;
1230
+ scan = p - 1;
1231
+ continue;
1232
+ }
1233
+ break;
1080
1234
  }
1081
1235
  return count;
1082
1236
  };
1083
1237
  /**
1238
+ * True when `word` is a recognized legal-form suffix
1239
+ * (case-sensitive against the legal-forms vocabulary).
1240
+ * Used when deciding whether to cross an "and" connector
1241
+ * during backward extension — if the word immediately
1242
+ * preceding the connector is itself a legal-form suffix,
1243
+ * the "and" sits between two organisation names rather
1244
+ * than inside one ("Morgan Securities LLC and Allen &
1245
+ * Company LLC"), so the walk must stop there.
1246
+ */
1247
+ const isKnownLegalFormSuffix = (word) => {
1248
+ if (word.length === 0) return false;
1249
+ return getNormalizedLegalBoundarySuffixesSync().has(word);
1250
+ };
1251
+ const isInNameLegalFormWord = (word) => {
1252
+ if (word.length === 0) return false;
1253
+ return getNormalizedInNameLegalFormWordsSync().has(word);
1254
+ };
1255
+ /**
1256
+ * If `pos` is immediately preceded (modulo horizontal
1257
+ * whitespace) by an initial-dot run like `J.P.`, `U.S.`,
1258
+ * or `N.A.`, return the position where the initial run
1259
+ * starts. Otherwise return `pos` unchanged. The run must
1260
+ * be word-bounded on the left so we never absorb a stray
1261
+ * sentence-ending dot.
1262
+ */
1263
+ const skipInitialsBackward = (fullText, pos) => {
1264
+ let scan = pos - 1;
1265
+ while (scan >= 0) {
1266
+ const ch = fullText.charAt(scan);
1267
+ if (ch === "\n" || !/\s/.test(ch)) break;
1268
+ scan--;
1269
+ }
1270
+ if (scan < 0 || fullText.charAt(scan) !== ".") return pos;
1271
+ const scanLimit = Math.max(0, scan + 1 - 100);
1272
+ const head = fullText.slice(scanLimit, scan + 1);
1273
+ const match = /(?:\p{Lu}\.[^\S\n]?){2,}$/u.exec(head);
1274
+ if (match === null) return pos;
1275
+ const start = scanLimit + match.index;
1276
+ if (start > 0) {
1277
+ const prevCh = fullText.charAt(start - 1);
1278
+ if (/[\p{L}\p{M}\p{N}]/u.test(prevCh)) return pos;
1279
+ }
1280
+ return start;
1281
+ };
1282
+ /**
1084
1283
  * Extend a match backward through uppercase words and
1085
1284
  * lowercase connectors. Stops at start of text,
1086
1285
  * newline, or a word that doesn't qualify.
@@ -1109,19 +1308,30 @@ const extendBackward = (fullText, matchStart, options = {}) => {
1109
1308
  const isConnector = CONNECTOR_RE.test(word);
1110
1309
  const isInNamePrep = suffixMode && IN_NAME_PREPOSITION_RE.test(word);
1111
1310
  if (isUpper) pos = wordStart;
1112
- else if (isConnector) {
1113
- if (!suffixMode && AND_TYPE_CONNECTOR_RE.test(word) && countUpperWordsBefore(fullText, wordStart) === 2) break;
1311
+ else if (isConnector) if (AND_TYPE_CONNECTOR_RE.test(word)) {
1312
+ const prev = findWordBefore(fullText, wordStart);
1313
+ if (!prev) break;
1314
+ if (!UPPER_LETTER_RE.test(prev.word)) break;
1315
+ if (isKnownLegalFormSuffix(prev.word)) break;
1316
+ const upperWordsBefore = countUpperWordsBefore(fullText, wordStart, suffixMode);
1317
+ const middleInitialBefore = hasMiddleInitialBefore(fullText, wordStart);
1318
+ if (upperWordsBefore <= 1 && (getClauseNounHeadsSync().has(prev.word.toLowerCase()) || getConnectorProseHeadsSync().has(prev.word.toLowerCase()))) break;
1319
+ if (suffixMode ? middleInitialBefore && hasSingleCapPrefixBefore(fullText, matchStart) : upperWordsBefore === 2 && !isInNameLegalFormWord(prev.word) || middleInitialBefore) break;
1320
+ pos = prev.start;
1321
+ } else {
1114
1322
  const prev = findWordBefore(fullText, wordStart);
1115
1323
  if (!prev) break;
1116
1324
  if (!UPPER_LETTER_RE.test(prev.word)) break;
1117
1325
  pos = prev.start;
1118
- } else if (isInNamePrep) {
1326
+ }
1327
+ else if (isInNamePrep) {
1119
1328
  const prev = findWordBefore(fullText, wordStart);
1120
1329
  if (!prev) break;
1121
1330
  if (!UPPER_LETTER_RE.test(prev.word)) break;
1122
1331
  pos = prev.start;
1123
1332
  } else break;
1124
1333
  }
1334
+ pos = skipInitialsBackward(fullText, pos);
1125
1335
  return pos;
1126
1336
  };
1127
1337
  const trimLeadingClause = (text) => {
@@ -1162,6 +1372,7 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1162
1372
  const firstWordMatch = /^[\p{L}\p{M}]+(?:-[\p{L}\p{M}]+)*/u.exec(text);
1163
1373
  let processedStart = match.start;
1164
1374
  let processedText = text;
1375
+ if (isStructuralSingleCapMatch(processedText) || fullText !== void 0 && isBareSingleCapStructuralInnerMatch(fullText, match.start, text)) continue;
1165
1376
  let trimmed = false;
1166
1377
  const firstWordText = firstWordMatch?.[0] ?? "";
1167
1378
  const firstWordLeading = /^[\p{L}\p{M}]+/u.exec(firstWordText)?.[0] ?? "";
@@ -1212,12 +1423,15 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
1212
1423
  let entityStart = processedStart;
1213
1424
  let entityText = processedText;
1214
1425
  if (fullText && !trimmed) {
1215
- const extended = extendBackward(fullText, processedStart);
1426
+ const extended = !BARE_SINGLE_CAP_LEGAL_FORM_RE.test(processedText) ? extendBackward(fullText, processedStart) : processedStart;
1216
1427
  if (extended < processedStart) {
1217
1428
  entityStart = extended;
1218
1429
  entityText = fullText.slice(extended, processedStart + processedText.length).trimEnd();
1219
1430
  }
1220
1431
  }
1432
+ const listTrim = trimEmbeddedLegalFormListPrefix(entityStart, entityText);
1433
+ entityStart = listTrim.entityStart;
1434
+ entityText = listTrim.entityText;
1221
1435
  const clauseTrim = trimLeadingClause(entityText);
1222
1436
  if (clauseTrim.offset > 0) {
1223
1437
  entityStart += clauseTrim.offset;
@@ -1564,20 +1778,23 @@ const processGazetteerMatches = (allMatches, sliceStart, sliceEnd, fullText, dat
1564
1778
  const label = data.labels[localIdx];
1565
1779
  if (!label) continue;
1566
1780
  const extended = tryPrefixExtension(fullText, match.start, match.end);
1781
+ const isExtended = extended !== null;
1567
1782
  const end = extended?.end ?? match.end;
1568
1783
  const text = extended?.text ?? fullText.slice(match.start, match.end);
1569
1784
  exactSpans.push({
1570
1785
  start: match.start,
1571
1786
  end
1572
1787
  });
1573
- results.push({
1788
+ const entity = {
1574
1789
  start: match.start,
1575
1790
  end,
1576
1791
  label,
1577
1792
  text,
1578
1793
  score: .9,
1579
1794
  source: DETECTION_SOURCES.GAZETTEER
1580
- });
1795
+ };
1796
+ if (isExtended) entity.sourceDetail = "gazetteer-extension";
1797
+ results.push(entity);
1581
1798
  }
1582
1799
  for (const match of allMatches) {
1583
1800
  const idx = match.pattern;
@@ -2122,8 +2339,10 @@ const MIN_MONTH_NAME_LENGTH = 3;
2122
2339
  const escapeTitle = (title) => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\s+/g, "\\s*");
2123
2340
  /** Escape for use inside a regex alternation. */
2124
2341
  const escapeRegex$2 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
2342
+ const escapeRegexPhrase = (s) => escapeRegex$2(s.trim()).replace(/\s+/g, "[^\\S\\n\\t]+");
2125
2343
  /** Escape for use inside a regex character class. */
2126
2344
  const escapeCharClass = (s) => s.replace(/[\]\\^-]/g, "\\$&");
2345
+ const toSortedAlternation = (values) => [...new Set(values.map(escapeRegexPhrase).filter((value) => value.length > 0))].toSorted((a, b) => b.length - a.length).join("|");
2127
2346
  const TITLE_PREFIX = TITLE_PREFIXES.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
2128
2347
  const POST_NOMINAL = POST_NOMINALS.toSorted((a, b) => b.length - a.length).map(escapeTitle).join("|");
2129
2348
  const NAME_WORD = `\\p{Lu}\\p{Ll}+`;
@@ -2134,6 +2353,7 @@ const HONORIFIC_ALT = [...HONORIFICS].toSorted((a, b) => b.length - a.length).ma
2134
2353
  const escaped = escapeRegex$2(h);
2135
2354
  return HONORIFIC_BOUNDARY.has(h) ? `\\b${escaped}` : escaped;
2136
2355
  }).join("|");
2356
+ const AMOUNT_WORDS = amount_words_default;
2137
2357
  const toEntry = (validator, label, score) => {
2138
2358
  const pattern = toRegex(validator).source;
2139
2359
  if (!pattern) return null;
@@ -2347,6 +2567,58 @@ const SHORT_TLDS = "de|at|be|se|fi|dk|no|it|uk";
2347
2567
  const HOST_LABEL = `[a-zA-Z0-9](?:[a-zA-Z0-9\\-]*[a-zA-Z0-9])?`;
2348
2568
  const BARE_HOST = `\\b[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]`;
2349
2569
  const PATH_SUFFIX = `(?:[/?#][^\\s)\\]>]*[^\\s.,;:!?)\\]>])?`;
2570
+ const BARE_DOMAIN = {
2571
+ pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
2572
+ label: "url",
2573
+ score: .9
2574
+ };
2575
+ const IPV6_ADDRESS = {
2576
+ pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
2577
+ label: "ip address",
2578
+ score: 1
2579
+ };
2580
+ const MAC_ADDRESS = {
2581
+ pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
2582
+ label: "mac address",
2583
+ score: 1
2584
+ };
2585
+ const UK_POSTCODE = {
2586
+ pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
2587
+ label: "address",
2588
+ score: .9
2589
+ };
2590
+ const UK_NINO = {
2591
+ pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
2592
+ label: "social security number",
2593
+ score: .95,
2594
+ validator: gb.nino
2595
+ };
2596
+ const TIME_12H = {
2597
+ pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
2598
+ label: "date",
2599
+ score: .9
2600
+ };
2601
+ const PERCENT_TOKEN = `${`(?:[+${DASH_INNER}])?(?:\\d{1,3}(?:[.,]\\d{3})+(?:[.,]\\d{1,4})?|\\d+(?:[.,]\\d{1,4})?)`}[^\\S\\n]{0,2}%`;
2602
+ const PERCENT_RANGE_NUMBER = `\\d+(?:[.,]\\d{1,4})?`;
2603
+ const PERCENT_RANGE = `${PERCENT_RANGE_NUMBER}[^\\S\\n]*${DASH}[^\\S\\n]*${PERCENT_RANGE_NUMBER}[^\\S\\n]{0,2}%`;
2604
+ const buildPercentWordPattern = (config) => {
2605
+ const phrases = [];
2606
+ for (const entry of config.percentages ?? []) {
2607
+ const ones = entry.ones.map(escapeRegex$2);
2608
+ const standalone = (entry.standalone ?? []).map(escapeRegex$2);
2609
+ const baseWords = [
2610
+ ...ones,
2611
+ ...entry.teens,
2612
+ ...entry.tens
2613
+ ].map(escapeRegex$2);
2614
+ const compoundSeparator = entry.allowSpaceCompoundSeparator ? `(?:${DASH}|[^\\S\\n]+)` : DASH;
2615
+ const compound = `(?:${baseWords.join("|")})` + (ones.length > 0 ? `(?:${compoundSeparator}(?:${ones.join("|")}))?` : "");
2616
+ const word = `(?:${[...standalone, compound].join("|")})`;
2617
+ const keyword = `(?:${entry.keywords.map(escapeRegex$2).join("|")})`;
2618
+ phrases.push(`${word}[^\\S\\n]+${keyword}`);
2619
+ }
2620
+ return phrases.length > 0 ? `(?i:(?:${phrases.join("|")}))` : "(?!)";
2621
+ };
2350
2622
  /**
2351
2623
  * All static PII regex definitions. Scanned in a
2352
2624
  * single pass by @stll/regex-set (Rust DFA).
@@ -2388,36 +2660,16 @@ const ALL_REGEX_DEFS = [
2388
2660
  BR_RG_WITH_SSP,
2389
2661
  BR_OAB,
2390
2662
  URL,
2663
+ IPV6_ADDRESS,
2664
+ MAC_ADDRESS,
2665
+ BARE_DOMAIN,
2666
+ UK_POSTCODE,
2667
+ UK_NINO,
2668
+ TIME_12H,
2391
2669
  {
2392
- pattern: "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|\\b(?:[0-9a-fA-F]{1,4}:){1,7}:\\b|::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}",
2393
- label: "ip address",
2394
- score: 1
2395
- },
2396
- {
2397
- pattern: "\\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\\b|\\b(?:[0-9a-fA-F]{2}-){5}[0-9a-fA-F]{2}\\b",
2398
- label: "mac address",
2399
- score: 1
2400
- },
2401
- {
2402
- pattern: `${BARE_HOST}(?:\\.${HOST_LABEL})*\\.(?:${LONG_TLDS})\\b${PATH_SUFFIX}|${BARE_HOST}(?:\\.${HOST_LABEL})+\\.(?:${SHORT_TLDS})\\b${PATH_SUFFIX}`,
2403
- label: "url",
2404
- score: .9
2405
- },
2406
- {
2407
- pattern: "\\b(?:GIR[^\\S\\n]?0AA|[A-PR-UWYZ](?:[A-HK-Y][0-9](?:[0-9]|[ABEHMNPRV-Y])?|[0-9](?:[0-9]|[A-HJKPS-UW])?)[^\\S\\n]?[0-9][ABD-HJLNP-UW-Z]{2})\\b",
2408
- label: "address",
2409
- score: .9
2410
- },
2411
- {
2412
- pattern: "\\b[A-CEGHJ-PR-TWXYZ][A-CEGHJ-NPR-TWXYZ][^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?\\d{2}[^\\S\\n]?[A-D]?\\b",
2413
- label: "social security number",
2414
- score: .95,
2415
- validator: gb.nino
2416
- },
2417
- {
2418
- pattern: "\\b(?:1[0-2]|0?[1-9]):[0-5]\\d[^\\S\\n]?(?:[aApP]\\.?[mM]\\.?)(?=[\\s,;!?)]|$)",
2419
- label: "date",
2420
- score: .9
2670
+ pattern: `(?<![\\p{L}\\p{N}_.,])(?:${buildPercentWordPattern(AMOUNT_WORDS)}[^\\S\\n]*\\([^\\S\\n]*${PERCENT_TOKEN}[^\\S\\n]*\\)|${PERCENT_RANGE}|${PERCENT_TOKEN})(?![\\p{L}\\p{N}_])`,
2671
+ label: "monetary amount",
2672
+ score: .85
2421
2673
  },
2422
2674
  ...STDNUM_ENTRIES
2423
2675
  ];
@@ -2491,6 +2743,41 @@ const DATE_PATTERN_META = Object.freeze({
2491
2743
  label: "date",
2492
2744
  score: 1
2493
2745
  });
2746
+ const buildMagnitudePattern = (config) => {
2747
+ const words = [];
2748
+ const caseInsensitiveAbbreviations = [];
2749
+ const caseSensitiveAbbreviations = [];
2750
+ for (const entry of config.magnitudeSuffixes ?? []) {
2751
+ words.push(...entry.words ?? []);
2752
+ caseInsensitiveAbbreviations.push(...entry.abbreviationsCaseInsensitive ?? []);
2753
+ caseSensitiveAbbreviations.push(...entry.abbreviationsCaseSensitive ?? []);
2754
+ }
2755
+ const branches = [];
2756
+ const wordsAlt = toSortedAlternation(words);
2757
+ const abbreviationCiAlt = toSortedAlternation(caseInsensitiveAbbreviations);
2758
+ const abbreviationCsAlt = toSortedAlternation(caseSensitiveAbbreviations);
2759
+ if (wordsAlt) branches.push(`[^\\S\\n\\t]+(?i:(?:${wordsAlt}))\\b`);
2760
+ if (abbreviationCiAlt) branches.push(`[^\\S\\n\\t]?(?i:${abbreviationCiAlt})\\b`);
2761
+ if (abbreviationCsAlt) branches.push(`[^\\S\\n\\t]?(?:${abbreviationCsAlt})\\b`);
2762
+ return branches.length > 0 ? `(?:${branches.join("|")})?` : "";
2763
+ };
2764
+ const buildQuantityFollowerGuard = (config) => {
2765
+ const modifiers = [];
2766
+ const nouns = [];
2767
+ for (const entry of config.shareQuantityTerms ?? []) {
2768
+ modifiers.push(...entry.modifiers ?? []);
2769
+ nouns.push(...entry.nouns);
2770
+ }
2771
+ const modifierAlt = toSortedAlternation(modifiers);
2772
+ const nounAlt = toSortedAlternation(nouns);
2773
+ if (!nounAlt) return "";
2774
+ return `(?![^\\S\\n\\t]+(?i:${modifierAlt ? `(?:(?:${modifierAlt})[^\\S\\n\\t]+){0,3}` : ""}(?:${nounAlt}))\\b)`;
2775
+ };
2776
+ const buildFinancialLexicons = (config) => Object.freeze({
2777
+ magnitude: buildMagnitudePattern(config),
2778
+ quantityFollowerGuard: buildQuantityFollowerGuard(config)
2779
+ });
2780
+ const FINANCIAL_LEXICONS = buildFinancialLexicons(AMOUNT_WORDS);
2494
2781
  /**
2495
2782
  * Build symbol character class, code alternation,
2496
2783
  * and local-name alternation from currencies.json,
@@ -2530,9 +2817,13 @@ const buildCurrencyPatterns = (data) => {
2530
2817
  const patterns = [];
2531
2818
  const DECIMAL = `(?:[.,](?=\\d|[${DASH_INNER}])[^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2}))?`;
2532
2819
  const END = `(?:\\b|(?=\\s|[.,;!?)]|$))`;
2533
- if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${END}`);
2534
- if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${END}`);
2535
- if (trailingAlt) patterns.push(`\\b${NUM}${DECIMAL}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${END}`);
2820
+ const MAGNITUDE = FINANCIAL_LEXICONS.magnitude;
2821
+ if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}${DECIMAL}${MAGNITUDE}${END}`);
2822
+ if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}${DECIMAL}${MAGNITUDE}${END}`);
2823
+ if (trailingAlt) {
2824
+ const optionalLeadingSymbol = symbols ? `(?<![\\p{L}\\p{N}_])(?:[${symbols}][^\\S\\n\\t]?)?` : "\\b";
2825
+ patterns.push(`${optionalLeadingSymbol}${NUM}${DECIMAL}${MAGNITUDE}[^\\S\\n\\t]{0,4}(?:${trailingAlt})${FINANCIAL_LEXICONS.quantityFollowerGuard}${END}`);
2826
+ }
2536
2827
  return patterns;
2537
2828
  };
2538
2829
  /** Cached promise for currency patterns. Loaded once. */
@@ -2846,10 +3137,19 @@ const stripQuotes = (value) => {
2846
3137
  text: stripped
2847
3138
  };
2848
3139
  };
2849
- /** Hard stop characters for to-next-comma scanning. */
3140
+ /**
3141
+ * Hard stop characters for to-next-comma scanning. A closing
3142
+ * parenthesis or closing bracket terminates a clause just like
3143
+ * an opening one: `State of New York or any other jurisdiction)`
3144
+ * is the tail of a parenthesised insertion, not the start of a
3145
+ * larger phrase that should be absorbed into a jurisdiction span.
3146
+ */
2850
3147
  const COMMA_STOP_CHARS = new Set([
2851
3148
  "\n",
2852
3149
  "(",
3150
+ ")",
3151
+ "[",
3152
+ "]",
2853
3153
  " ",
2854
3154
  ";"
2855
3155
  ]);
@@ -3015,6 +3315,27 @@ const getAddressStopKeywordsSync = () => addressStopKeywordsCache ?? ADDRESS_STO
3015
3315
  const warmAddressStopKeywords = async () => {
3016
3316
  await loadAddressStopKeywords();
3017
3317
  };
3318
+ const MAX_TRIGGER_VALUE_LEN = 100;
3319
+ const MIN_TRIGGER_PHONE_DIGITS = 5;
3320
+ const PHONE_VALUE_START_RE = /^[+(\d]/;
3321
+ const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
3322
+ const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
3323
+ const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
3324
+ const capAtWordBoundary = (valueText, cap) => {
3325
+ let capped = cap;
3326
+ const isWordChar = (i) => /[\p{L}\p{N}]/u.test(valueText[i] ?? "");
3327
+ while (capped > 0 && isWordChar(capped - 1) && isWordChar(capped)) capped--;
3328
+ return capped;
3329
+ };
3330
+ const isPlausiblePhoneTriggerValue = (value) => {
3331
+ const trimmed = value.trimStart();
3332
+ if (!PHONE_VALUE_START_RE.test(trimmed)) return false;
3333
+ if (ISO_DATE_PREFIX_RE.test(trimmed)) return false;
3334
+ if (INLINE_FIELD_LABEL_RE.test(trimmed)) return false;
3335
+ let digits = 0;
3336
+ for (const ch of trimmed) if (/\d/.test(ch)) digits++;
3337
+ return digits >= MIN_TRIGGER_PHONE_DIGITS;
3338
+ };
3018
3339
  const extractValue = (text, triggerEnd, strategy, label) => {
3019
3340
  const remaining = text.slice(triggerEnd);
3020
3341
  const stripped = remaining.replace(/^[\s:;]+/, "");
@@ -3026,21 +3347,11 @@ const extractValue = (text, triggerEnd, strategy, label) => {
3026
3347
  const stopWords = strategy.stopWords ?? [];
3027
3348
  const stopWordRe = stopWords.length > 0 ? new RegExp(`^(?:${stopWords.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?![\\p{L}\\p{N}])`, "iu") : null;
3028
3349
  let end = 0;
3029
- let foundStop = false;
3030
3350
  while (end < valueText.length) {
3031
3351
  const ch = valueText[end];
3032
- if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) {
3033
- foundStop = true;
3034
- break;
3035
- }
3036
- if (ch === "." && isSentenceTerminator(valueText, end)) {
3037
- foundStop = true;
3038
- break;
3039
- }
3040
- if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) {
3041
- foundStop = true;
3042
- break;
3043
- }
3352
+ if (ch !== void 0 && COMMA_STOP_CHARS.has(ch)) break;
3353
+ if (ch === "." && isSentenceTerminator(valueText, end)) break;
3354
+ if (stopWordRe !== null && (end === 0 || !/[\p{L}\p{N}]/u.test(valueText[end - 1] ?? "")) && stopWordRe.test(valueText.slice(end))) break;
3044
3355
  if (ch === ",") {
3045
3356
  const afterComma = valueText.slice(end);
3046
3357
  if (DECIMAL_COMMA_RE.test(afterComma)) {
@@ -3052,12 +3363,12 @@ const extractValue = (text, triggerEnd, strategy, label) => {
3052
3363
  end += degreeMatch[0].length;
3053
3364
  continue;
3054
3365
  }
3055
- foundStop = true;
3056
3366
  break;
3057
3367
  }
3058
3368
  end++;
3059
3369
  }
3060
- if (!foundStop) end = Math.min(end, 100);
3370
+ const lengthCap = strategy.maxLength ?? 100;
3371
+ if (end > lengthCap) end = capAtWordBoundary(valueText, lengthCap);
3061
3372
  const rawSlice = valueText.slice(0, end);
3062
3373
  const extracted = rawSlice.trim();
3063
3374
  if (extracted.length === 0) return null;
@@ -3071,12 +3382,24 @@ const extractValue = (text, triggerEnd, strategy, label) => {
3071
3382
  case "to-end-of-line": {
3072
3383
  const consumed = remaining.length - valueText.length;
3073
3384
  if (consumed > 0 && remaining.slice(0, consumed).includes("\n")) return null;
3074
- const LINE_STOPS = ["\n"];
3385
+ const LINE_STOPS = ["\n", " "];
3075
3386
  let end = valueText.length;
3387
+ let foundLineStop = false;
3076
3388
  for (const ch of LINE_STOPS) {
3077
3389
  const idx = valueText.indexOf(ch);
3078
- if (idx !== -1 && idx < end) end = idx;
3390
+ if (idx !== -1 && idx < end) {
3391
+ end = idx;
3392
+ foundLineStop = true;
3393
+ }
3394
+ }
3395
+ if (label === "phone number") {
3396
+ const inlineLabel = INLINE_FIELD_LABEL_STOP_RE.exec(valueText.slice(0, end));
3397
+ if (inlineLabel) {
3398
+ end = inlineLabel.index;
3399
+ foundLineStop = true;
3400
+ }
3079
3401
  }
3402
+ if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
3080
3403
  const rawSlice = valueText.slice(0, end);
3081
3404
  const extracted = rawSlice.trim();
3082
3405
  if (extracted.length === 0) return null;
@@ -3256,6 +3579,7 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
3256
3579
  const value = rawValue ? stripQuotes(rawValue) : null;
3257
3580
  if (value) {
3258
3581
  if (!applyValidations(value.text, rule.validations)) continue;
3582
+ if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
3259
3583
  const entityStart = rule.includeTrigger ? match.start : value.start;
3260
3584
  const entityEnd = value.end;
3261
3585
  const entityText = fullText.slice(entityStart, entityEnd);
@@ -3597,7 +3921,7 @@ const normalizeHomoglyphs = (text) => {
3597
3921
  //#endregion
3598
3922
  //#region src/filters/false-positives.ts
3599
3923
  const TEMPLATE_PLACEHOLDER_RE = /^(?:\.{3,}|_{3,}|\[[\w\s]+\]|\{[\w\s]+\})$/;
3600
- const POSTAL_CODE_RE = /\d{3}\s?\d{2}/;
3924
+ const POSTAL_CODE_RE$1 = /\d{3}\s?\d{2}/;
3601
3925
  const HAS_DIGIT_RE = /\d/;
3602
3926
  const ADDRESS_COMPONENT_EXTRA_RE = /(?:^|\s)(?:č\.p\.|č\.ev\.|č\.|sídliště)(?=[\s,./]|$)/i;
3603
3927
  const BARE_COURS_PROSE_RE = /(?:^|\s)cours(?!\s+\p{Lu})(?=[\s,./]|$)/u;
@@ -3662,7 +3986,7 @@ const normalizeEntity = (entity) => {
3662
3986
  text
3663
3987
  };
3664
3988
  };
3665
- const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
3989
+ const EMPTY_GENERIC_ROLES$1 = /* @__PURE__ */ new Set();
3666
3990
  /**
3667
3991
  * Load generic-roles.json and cache the result on the
3668
3992
  * given context. Must be awaited during pipeline init
@@ -3686,14 +4010,14 @@ const loadGenericRoles = (ctx = defaultContext) => {
3686
4010
  return ctx.genericRolesPromise;
3687
4011
  };
3688
4012
  /** Sync accessor — returns empty set before init. */
3689
- const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
4013
+ const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES$1;
3690
4014
  const STREET_TYPES_SEED_RE = /(?:^|\s)(?:ul\.|ulice|nám\.|náměstí|tř\.|třída|nábř\.|nábřeží|bulvár)(?=[\s,./]|$)/i;
3691
4015
  let _streetTypesRe = STREET_TYPES_SEED_RE;
3692
4016
  let _streetTypesPromise = null;
3693
4017
  const escapeRegex$1 = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3694
4018
  const loadStreetTypeRegex = async () => {
3695
4019
  try {
3696
- const data = (await import("./address-street-types.mjs")).default ?? {};
4020
+ const data = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
3697
4021
  const words = /* @__PURE__ */ new Set();
3698
4022
  for (const [key, val] of Object.entries(data)) {
3699
4023
  if (key.startsWith("_")) continue;
@@ -3762,7 +4086,7 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
3762
4086
  }
3763
4087
  if ((normalized.label === "person" || normalized.label === "organization") && roles.has(normalizeHomoglyphs(trimmed).toLowerCase())) continue;
3764
4088
  if (normalized.label === "organization" && normalized.source === "legal-form" && trimmed === trimmed.toUpperCase() && LEGAL_FORM_HEADING_RE.test(trimmed)) continue;
3765
- if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
4089
+ if (normalized.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE$1.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3766
4090
  if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !hasAddressComponent(trimmed) && !JURISDICTION_RE.test(trimmed)) continue;
3767
4091
  if (normalized.label === "address" && normalized.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && isOnlyAmbiguousCours(trimmed)) continue;
3768
4092
  if (normalized.label === "address" && SIGNING_CLAUSE_ADDRESS_RE.test(trimmed)) continue;
@@ -3827,6 +4151,23 @@ const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
3827
4151
  /** Sync accessor — returns empty set before init. */
3828
4152
  const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
3829
4153
  /**
4154
+ * Curated dictionary entries that are pure dotted
4155
+ * single-letter acronyms (e.g. `S.C.`, `D.N.J.`, `C.E.C.`)
4156
+ * need targeted suffix guards. The AC search matches
4157
+ * case-insensitively on token boundaries where `.` is not
4158
+ * a word character, so `S.C.` can match inside `U.S.C.`.
4159
+ * Two-segment non-address aliases are too noisy and are
4160
+ * dropped at build time; longer official aliases stay
4161
+ * searchable and are only suppressed when the source text
4162
+ * shows they are the tail of a longer dotted token.
4163
+ * Caller-supplied custom entries are exempted.
4164
+ */
4165
+ const DOTTED_ACRONYM_RE = /^(?=.{3,}$)\p{L}(?:\.\p{L}){0,3}\.?$/u;
4166
+ const isCuratedNoiseAcronym = (normalized) => DOTTED_ACRONYM_RE.test(normalized);
4167
+ const dottedAcronymSegmentCount = (normalized) => normalized.split(".").filter(Boolean).length;
4168
+ const isShortCuratedNoiseAcronym = (normalized) => isCuratedNoiseAcronym(normalized) && dottedAcronymSegmentCount(normalized) <= 2;
4169
+ const isDottedAcronymSuffixCollision = (fullText, start, matchText) => isCuratedNoiseAcronym(matchText) && /[\p{L}]\.$/u.test(fullText.slice(Math.max(0, start - 2), start));
4170
+ /**
3830
4171
  * Common EU given names present in the stopwords-iso dataset
3831
4172
  * but absent from the first-name corpus. Without this
3832
4173
  * supplementary set, these names would pass through the
@@ -3962,7 +4303,7 @@ let streetTypeReLoaded = false;
3962
4303
  const loadStreetTypeRe = async () => {
3963
4304
  if (streetTypeReLoaded) return cachedStreetTypeRe;
3964
4305
  try {
3965
- const config = (await import("./address-street-types.mjs")).default ?? {};
4306
+ const config = (await import("./address-street-types.mjs").then((n) => n.n)).default ?? {};
3966
4307
  const words = [];
3967
4308
  for (const value of Object.values(config)) {
3968
4309
  if (!Array.isArray(value)) continue;
@@ -4076,6 +4417,7 @@ const buildDenyList = async (config, ctx = defaultContext) => {
4076
4417
  const addDenyListEntry = (entry, label, source = "deny-list") => {
4077
4418
  const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
4078
4419
  if (normalized.length === 0) return;
4420
+ if (source !== "custom-deny-list" && label !== "address" && isShortCuratedNoiseAcronym(normalized)) return;
4079
4421
  const lower = normalized.toLowerCase();
4080
4422
  const existing = patternIndex.get(lower);
4081
4423
  if (existing !== void 0) {
@@ -4153,6 +4495,7 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
4153
4495
  const addNameEntry = (name, source) => {
4154
4496
  const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
4155
4497
  if (normalized.length === 0) return;
4498
+ if (isCuratedNoiseAcronym(normalized)) return;
4156
4499
  const lower = normalized.toLowerCase();
4157
4500
  const existing = patternIndex.get(lower);
4158
4501
  if (existing !== void 0) {
@@ -4239,12 +4582,14 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
4239
4582
  const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
4240
4583
  const customLabels = customEdgesAreValid ? customPatternLabels : [];
4241
4584
  if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
4242
- const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
4243
- if (curatedLabels.length === 0 && customLabels.length === 0) continue;
4585
+ const acronymMatchesAcronym = !(pattern.length > 0 && pattern.length <= 5 && ALL_UPPER_RE.test(pattern)) || ALL_UPPER_RE.test(matchText);
4586
+ const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && acronymMatchesAcronym && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
4587
+ const filteredCuratedLabels = isDottedAcronymSuffixCollision(fullText, match.start, matchText) ? [] : curatedLabels;
4588
+ if (filteredCuratedLabels.length === 0 && customLabels.length === 0) continue;
4244
4589
  const entry = {
4245
4590
  start: match.start,
4246
4591
  end: match.end,
4247
- labels: curatedLabels,
4592
+ labels: filteredCuratedLabels,
4248
4593
  customLabels,
4249
4594
  sources,
4250
4595
  text: matchText,
@@ -4310,6 +4655,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
4310
4655
  const first = chain.at(0);
4311
4656
  const last = chain.at(-1);
4312
4657
  if (!first || !last) continue;
4658
+ if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
4313
4659
  const extended = extendPersonName(fullText, first.start, last.end, ctx);
4314
4660
  const score = chain.length >= 2 ? .9 : .5;
4315
4661
  if (chain.length === 1) {
@@ -4405,6 +4751,95 @@ const extendCityDistricts = (entities, fullText) => {
4405
4751
  * by a capitalized word (for "Miroslav Braňka" when
4406
4752
  * only "Braňka" matched).
4407
4753
  */
4754
+ /**
4755
+ * Defined-term marker: an opening typographic or straight
4756
+ * quote enclosing the chain start, AND a closing quote
4757
+ * within a short window followed by a
4758
+ * definitional cue (`means`, `shall mean`, `shall have
4759
+ * the meaning(s)`, `refers to`). Legal documents reserve
4760
+ * this construction for defined terms; the contents are
4761
+ * not personal names even when individual tokens collide
4762
+ * with the name corpus.
4763
+ *
4764
+ * Plain quotations like `"John Unknown" said ...` do NOT
4765
+ * count: there is no definitional cue, so the trailing
4766
+ * surname extension is still allowed to absorb `Unknown`.
4767
+ */
4768
+ const OPENING_QUOTES = new Set([
4769
+ "\"",
4770
+ "'",
4771
+ "“",
4772
+ "„",
4773
+ "‟",
4774
+ "‘",
4775
+ "‛",
4776
+ "«"
4777
+ ]);
4778
+ const CLOSING_QUOTES = new Set([
4779
+ "\"",
4780
+ "'",
4781
+ "”",
4782
+ "’",
4783
+ "»",
4784
+ "“"
4785
+ ]);
4786
+ const DEFINED_TERM_CUE_RE = /^[\s,]*(?:means?|shall\s+means?|shall\s+have\s+the\s+meanings?|refers?\s+to|has\s+the\s+meanings?|is\s+defined)\b/iu;
4787
+ const DEFINED_TERM_LOOKAHEAD = 120;
4788
+ const DEFINED_TERM_LOOKBEHIND = 80;
4789
+ const EMPTY_GENERIC_ROLES = /* @__PURE__ */ new Set();
4790
+ const isLetter = (ch) => ch !== void 0 && /^\p{L}$/u.test(ch);
4791
+ const isApostropheInsideWord = (text, index) => isLetter(text[index - 1]) && isLetter(text[index + 1]);
4792
+ const isQuoteBoundary = (text, index) => {
4793
+ const ch = text[index];
4794
+ if (ch !== "'" && ch !== "’") return true;
4795
+ return !isApostropheInsideWord(text, index);
4796
+ };
4797
+ const findDefinedTermQuoteContent = (text, start) => {
4798
+ const min = Math.max(0, start - DEFINED_TERM_LOOKBEHIND);
4799
+ let quoteStart = -1;
4800
+ for (let i = start - 1; i >= min; i--) {
4801
+ const ch = text[i];
4802
+ if (ch === "\n") break;
4803
+ if (ch && OPENING_QUOTES.has(ch) && isQuoteBoundary(text, i)) {
4804
+ quoteStart = i;
4805
+ break;
4806
+ }
4807
+ if (ch && CLOSING_QUOTES.has(ch) && isQuoteBoundary(text, i)) break;
4808
+ }
4809
+ if (quoteStart === -1) return null;
4810
+ const max = Math.min(text.length, quoteStart + 1 + DEFINED_TERM_LOOKAHEAD);
4811
+ for (let i = start; i < max; i++) {
4812
+ const ch = text[i];
4813
+ if (!ch || !CLOSING_QUOTES.has(ch) || !isQuoteBoundary(text, i)) continue;
4814
+ const after = text.slice(i + 1, max);
4815
+ if (!DEFINED_TERM_CUE_RE.test(after)) return null;
4816
+ return {
4817
+ content: text.slice(quoteStart + 1, i),
4818
+ afterClosingQuote: after
4819
+ };
4820
+ }
4821
+ return null;
4822
+ };
4823
+ const FIRST_WORD_RE = /^\p{L}+/u;
4824
+ const WORD_RE = /\p{L}+/gu;
4825
+ const startsWithKnownFirstName = (quoteContent, ctx) => {
4826
+ const firstWord = FIRST_WORD_RE.exec(quoteContent.trim())?.[0];
4827
+ if (!firstWord) return false;
4828
+ return new Set(getNameCorpusFirstNames(ctx).map((name) => name.toLowerCase())).has(firstWord.toLowerCase());
4829
+ };
4830
+ const hasPersonRoleDefinition = (afterClosingQuote, ctx) => {
4831
+ const roleWords = afterClosingQuote.replace(DEFINED_TERM_CUE_RE, "").match(WORD_RE)?.slice(0, 8) ?? [];
4832
+ if (roleWords.length === 0) return false;
4833
+ const genericRoles = ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
4834
+ return roleWords.some((word) => genericRoles.has(word.toLowerCase()));
4835
+ };
4836
+ const isSuppressibleDefinedTermQuote = (text, start, ctx) => {
4837
+ const definedTermQuote = findDefinedTermQuoteContent(text, start);
4838
+ if (definedTermQuote === null) return false;
4839
+ const words = definedTermQuote.content.match(WORD_RE) ?? [];
4840
+ if (words.length >= 2 && startsWithKnownFirstName(definedTermQuote.content, ctx) && hasPersonRoleDefinition(definedTermQuote.afterClosingQuote, ctx)) return false;
4841
+ return words.length >= 2;
4842
+ };
4408
4843
  const extendPersonName = (text, start, end, ctx) => {
4409
4844
  let newEnd = end;
4410
4845
  let pos = newEnd;
@@ -4415,7 +4850,7 @@ const extendPersonName = (text, start, end, ctx) => {
4415
4850
  if (!UPPER_START_RE.test(char)) break;
4416
4851
  let wordEnd = wordStart;
4417
4852
  while (wordEnd < text.length && !/\s/.test(text[wordEnd] ?? "")) wordEnd++;
4418
- const stripped = text.slice(wordStart, wordEnd).replace(/[,;.]+$/, "");
4853
+ const stripped = text.slice(wordStart, wordEnd).replace(/[,;.”"’'“»]+$/, "");
4419
4854
  if (stripped.length < 2) break;
4420
4855
  const lower = stripped.toLowerCase();
4421
4856
  if (getStopwords(ctx).has(lower) || getPersonStopwords(ctx).has(lower)) break;
@@ -4438,6 +4873,15 @@ const extendPersonName = (text, start, end, ctx) => {
4438
4873
  * cluster occasionally absorbs.
4439
4874
  */
4440
4875
  const ADDRESS_TRAILING_TRIM_RE = new RegExp(`[,;:\\s${OPENING_BRACKETS_INNER}${QUOTE_DOUBLE_INNER}${QUOTE_SINGLE_INNER}′]`, "u");
4876
+ const POSTAL_ADJACENT = `\\p{L}\\p{N}_${DASH_INNER}`;
4877
+ const POSTAL_CODE_RE = new RegExp(`(?<![${POSTAL_ADJACENT}])(?:\\d{3}\\s\\d{2}|\\d{2}${DASH}\\d{3}|\\d{5}${DASH}\\d{3}|\\d{5}${DASH}\\d{4})(?![${POSTAL_ADJACENT}])`, "gu");
4878
+ const BR_CEP_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{3}$`, "u");
4879
+ const US_ZIP_PLUS_FOUR_SHAPE_RE = new RegExp(`^\\d{5}${DASH}\\d{4}$`, "u");
4880
+ const US_STATE_ABBREV_BEFORE_ZIP_RE = new RegExp(`(?:^|[^A-Za-z0-9])(A[KLRZ]|C[AOT]|D[CE]|F[LM]|G[AU]|HI|I[ADLN]|K[SY]|LA|M[ADEHINOPST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\\s*,?\\s*$`, "u");
4881
+ const US_ZIP_CONTEXT_WINDOW = 120;
4882
+ const US_CITY_ZIP_GAP_RE = /^[\s,]+$/u;
4883
+ const HOUSE_NUMBER_BEFORE_STREET_RE = /\b\d{1,6}(?:[-/]\d{1,6})?\s+(?:\p{Lu}\p{L}+[^\S\n\t]+){0,4}$/u;
4884
+ const HOUSE_NUMBER_AFTER_STREET_RE = /^[^\S\n\t]+\d{1,6}(?:[-/]\d{1,6})?\b/u;
4441
4885
  let cachedBoundaryRe = null;
4442
4886
  const loadBoundaryWords = async () => {
4443
4887
  try {
@@ -4452,7 +4896,7 @@ let cachedBrCepContextPromise = null;
4452
4896
  const loadBrCueWords = async () => {
4453
4897
  const sources = await Promise.all([(async () => {
4454
4898
  try {
4455
- return (await import("./address-street-types.mjs")).default["pt-br"];
4899
+ return (await import("./address-street-types.mjs").then((n) => n.n)).default["pt-br"];
4456
4900
  } catch {
4457
4901
  return;
4458
4902
  }
@@ -4496,6 +4940,48 @@ const hasBrCueNearby = (fullText, start, end, re) => {
4496
4940
  const window = fullText.slice(windowStart, windowEnd);
4497
4941
  return new RegExp(re.source, re.flags.replace("g", "")).test(window);
4498
4942
  };
4943
+ const getUsStateSeedBeforeZip = (fullText, start) => {
4944
+ const stateWindowStart = Math.max(0, start - 24);
4945
+ const stateWindow = fullText.slice(stateWindowStart, start);
4946
+ const match = US_STATE_ABBREV_BEFORE_ZIP_RE.exec(stateWindow);
4947
+ const state = match?.[1];
4948
+ if (!match || !state) return null;
4949
+ const stateOffset = match[0].indexOf(state);
4950
+ const stateStart = stateWindowStart + match.index + stateOffset;
4951
+ return {
4952
+ type: "state",
4953
+ start: stateStart,
4954
+ end: stateStart + state.length,
4955
+ text: state
4956
+ };
4957
+ };
4958
+ const hasHouseNumberNearStreetWord = (fullText, seed) => {
4959
+ if (/\d/.test(seed.text)) return true;
4960
+ const before = fullText.slice(Math.max(0, seed.start - 50), seed.start);
4961
+ if (HOUSE_NUMBER_BEFORE_STREET_RE.test(before)) return true;
4962
+ const after = fullText.slice(seed.end, Math.min(fullText.length, seed.end + 24));
4963
+ return HOUSE_NUMBER_AFTER_STREET_RE.test(after);
4964
+ };
4965
+ const getUsZipPlusFourContext = (fullText, start, seeds) => {
4966
+ const stateSeed = getUsStateSeedBeforeZip(fullText, start);
4967
+ if (stateSeed !== null) return {
4968
+ stateSeed,
4969
+ hasContext: true
4970
+ };
4971
+ return {
4972
+ stateSeed: null,
4973
+ hasContext: seeds.some((seed) => {
4974
+ if (Math.abs(seed.start - start) > US_ZIP_CONTEXT_WINDOW) return false;
4975
+ if (seed.type === "address-trigger") return true;
4976
+ if (seed.type === "city" && seed.end <= start) {
4977
+ const gap = fullText.slice(seed.end, start);
4978
+ return US_CITY_ZIP_GAP_RE.test(gap);
4979
+ }
4980
+ if (seed.type === "street-word") return hasHouseNumberNearStreetWord(fullText, seed);
4981
+ return false;
4982
+ })
4983
+ };
4984
+ };
4499
4985
  /**
4500
4986
  * Build regex for boundary words. Matches any
4501
4987
  * boundary word preceded by a word boundary.
@@ -4520,7 +5006,7 @@ const getBoundaryRe = async () => {
4520
5006
  const buildStreetTypePatterns = async () => {
4521
5007
  let config = {};
4522
5008
  try {
4523
- config = (await import("./address-street-types.mjs")).default;
5009
+ config = (await import("./address-street-types.mjs").then((n) => n.n)).default;
4524
5010
  } catch {
4525
5011
  return [];
4526
5012
  }
@@ -4565,13 +5051,22 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
4565
5051
  text: e.text
4566
5052
  });
4567
5053
  }
4568
- const postalRe = /\b(?:\d{3}\s\d{2}|\d{2}-\d{3}|\d{5}-\d{3})\b/g;
5054
+ const postalRe = POSTAL_CODE_RE;
5055
+ postalRe.lastIndex = 0;
4569
5056
  let postalMatch;
4570
5057
  while ((postalMatch = postalRe.exec(fullText)) !== null) {
4571
5058
  const start = postalMatch.index;
4572
5059
  const end = start + postalMatch[0].length;
4573
5060
  if (seeds.some((s) => s.start <= start && s.end >= end)) continue;
4574
- if (/^\d{5}-\d{3}$/.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
5061
+ if (BR_CEP_SHAPE_RE.test(postalMatch[0]) && (brCepContextRe === null || !hasBrCueNearby(fullText, start, end, brCepContextRe))) continue;
5062
+ if (US_ZIP_PLUS_FOUR_SHAPE_RE.test(postalMatch[0])) {
5063
+ const usContext = getUsZipPlusFourContext(fullText, start, seeds);
5064
+ if (!usContext.hasContext) continue;
5065
+ const stateSeed = usContext.stateSeed;
5066
+ if (stateSeed !== null) {
5067
+ if (!seeds.some((seed) => seed.start === stateSeed.start && seed.end === stateSeed.end)) seeds.push(stateSeed);
5068
+ }
5069
+ }
4575
5070
  seeds.push({
4576
5071
  type: "postal-code",
4577
5072
  start,
@@ -4650,6 +5145,7 @@ const scoreCluster = (cluster) => {
4650
5145
  let score = .5;
4651
5146
  if (types.has("postal-code")) score += .15;
4652
5147
  if (types.has("city")) score += .15;
5148
+ if (types.has("state")) score += .15;
4653
5149
  if (types.has("street-word")) score += .15;
4654
5150
  if (types.has("address-trigger")) score += .1;
4655
5151
  return Math.min(score, .95);
@@ -4836,7 +5332,44 @@ const BARE_STOPWORDS = new Set([
4836
5332
  "Splatnost",
4837
5333
  "Variabilní",
4838
5334
  "Konstantní",
4839
- "Specifický"
5335
+ "Specifický",
5336
+ "Section",
5337
+ "Sections",
5338
+ "Article",
5339
+ "Articles",
5340
+ "Schedule",
5341
+ "Schedules",
5342
+ "Exhibit",
5343
+ "Exhibits",
5344
+ "Annex",
5345
+ "Annexes",
5346
+ "Appendix",
5347
+ "Appendices",
5348
+ "Clause",
5349
+ "Clauses",
5350
+ "Chapter",
5351
+ "Chapters",
5352
+ "Paragraph",
5353
+ "Paragraphs",
5354
+ "Subsection",
5355
+ "Subsections",
5356
+ "Form",
5357
+ "Page",
5358
+ "Pages",
5359
+ "Item",
5360
+ "Items",
5361
+ "Note",
5362
+ "Notes",
5363
+ "Rule",
5364
+ "Rules",
5365
+ "Attachment",
5366
+ "Attachments",
5367
+ "Volume",
5368
+ "Volumes",
5369
+ "Book",
5370
+ "Books",
5371
+ "Part",
5372
+ "Parts"
4840
5373
  ]);
4841
5374
  const NEAR_MISS_BAND = .15;
4842
5375
  const BOOST_PER_NEIGHBOUR = .05;
@@ -4909,21 +5442,23 @@ const initPrepositions = () => {
4909
5442
  };
4910
5443
  const getAddressPreps = () => _addressPreps ?? /* @__PURE__ */ new Set();
4911
5444
  const getTemporalPreps = () => _temporalPreps ?? /* @__PURE__ */ new Set();
4912
- let _streetAbbrevs = null;
5445
+ const buildStreetAbbrevs = (data) => {
5446
+ const abbrevs = /* @__PURE__ */ new Set();
5447
+ for (const [key, words] of Object.entries(data)) {
5448
+ if (key.startsWith("_")) continue;
5449
+ if (!Array.isArray(words)) continue;
5450
+ for (const word of words) if (typeof word === "string" && word.includes(".")) abbrevs.add(word.toLowerCase());
5451
+ }
5452
+ return abbrevs;
5453
+ };
5454
+ let _streetAbbrevs = buildStreetAbbrevs(address_street_types_default);
4913
5455
  let _streetAbbrevsPromise = null;
4914
5456
  const loadStreetAbbrevs = async () => {
4915
5457
  try {
4916
- const mod = await import("./address-street-types.mjs");
4917
- const data = mod.default ?? mod;
4918
- const abbrevs = /* @__PURE__ */ new Set();
4919
- for (const [key, words] of Object.entries(data)) {
4920
- if (key.startsWith("_")) continue;
4921
- if (!Array.isArray(words)) continue;
4922
- for (const w of words) if (w.includes(".")) abbrevs.add(w.toLowerCase());
4923
- }
4924
- _streetAbbrevs = abbrevs;
5458
+ const mod = await import("./address-street-types.mjs").then((n) => n.n);
5459
+ _streetAbbrevs = buildStreetAbbrevs(mod.default ?? mod);
4925
5460
  } catch {
4926
- _streetAbbrevs = /* @__PURE__ */ new Set();
5461
+ _streetAbbrevs ??= /* @__PURE__ */ new Set();
4927
5462
  }
4928
5463
  };
4929
5464
  /** Ensure street abbreviation data is loaded. */
@@ -5035,7 +5570,7 @@ const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
5035
5570
  }
5036
5571
  return results;
5037
5572
  };
5038
- const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]+)*\s+\d{2,4}[a-zA-Z]?)\s*$/gmu;
5573
+ const ORPHAN_STREET_RE = /^[^\S\n]*(\p{Lu}[\p{Ll}\p{Lu}]+(?:[^\S\n]+[\p{Lu}\p{Ll}][\p{Ll}]+)*[^\S\n]+\d{2,4}[a-zA-Z]?)[^\S\n]*$/gmu;
5039
5574
  /**
5040
5575
  * In the header zone (top 15%), find standalone lines
5041
5576
  * matching "[Uppercase word(s)] [number]" that sit
@@ -5413,6 +5948,7 @@ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-lis
5413
5948
  * of `\s` to avoid merging entities across newlines.
5414
5949
  */
5415
5950
  const GAP_PATTERN = /^[ \t,\-]+$/;
5951
+ const isLegalFormOrganization = (entity) => entity.label === "organization" && entity.source === DETECTION_SOURCES.LEGAL_FORM;
5416
5952
  /**
5417
5953
  * Build a set of word boundary offsets for the full
5418
5954
  * text using `Intl.Segmenter`. Returns a sorted array
@@ -5552,7 +6088,7 @@ const mergeAdjacent = (entities, fullText) => {
5552
6088
  break;
5553
6089
  }
5554
6090
  }
5555
- if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
6091
+ if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev) && isLegalFormOrganization(entity) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
5556
6092
  prev.end = entity.end;
5557
6093
  prev.text = fullText.slice(prev.start, prev.end);
5558
6094
  prev.score = Math.max(prev.score, entity.score);
@@ -5952,6 +6488,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
5952
6488
  const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
5953
6489
  const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
5954
6490
  const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
6491
+ const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
6492
+ const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
5955
6493
  const shouldReplace = (a, b) => {
5956
6494
  const aLen = a.end - a.start;
5957
6495
  const bLen = b.end - b.start;
@@ -5967,6 +6505,31 @@ const shouldReplace = (a, b) => {
5967
6505
  /** Labels where colons are structurally significant. */
5968
6506
  const COLON_LABELS = new Set(["ip address", "mac address"]);
5969
6507
  /**
6508
+ * Labels whose entities should have a trailing sentence
6509
+ * `.` stripped during sanitisation. Restricted to
6510
+ * proper-noun-style labels where a final period is
6511
+ * almost always the sentence terminator that ran into
6512
+ * the capture, not a structural part of the value.
6513
+ * Numeric labels (`date`, `date of birth`, `phone
6514
+ * number`, `monetary amount`, `time`) and `person`
6515
+ * stay out — German writes `21. März`, post-nominal
6516
+ * degrees write `M.Sc.`, times write `5:00 p.m.`, and
6517
+ * stripping the dot would corrupt those spans.
6518
+ */
6519
+ const PERIOD_STRIPPED_LABELS = new Set([
6520
+ "organization",
6521
+ "location",
6522
+ "address"
6523
+ ]);
6524
+ const ADDRESS_FINAL_TOKEN_RE = /(?:^|[\s,])([\p{L}\p{M}.]+\.)$/u;
6525
+ const LOCATION_FINAL_DOTTED_ABBREV_RE = /(?:^|[\s,])(?:\p{Lu}\.){2,}$/u;
6526
+ const hasKnownAddressFinalAbbrev = (text) => {
6527
+ const finalToken = ADDRESS_FINAL_TOKEN_RE.exec(text)?.[1];
6528
+ if (!finalToken) return false;
6529
+ return getStreetAbbrevs().has(finalToken.toLowerCase());
6530
+ };
6531
+ const hasLocationFinalAbbrev = (text) => LOCATION_FINAL_DOTTED_ABBREV_RE.test(text);
6532
+ /**
5970
6533
  * Labels whose detectors emit precise, evidence-backed spans. When
5971
6534
  * one of these fires at the exact same offsets as a fuzzier
5972
6535
  * `address` hit (city dictionary lookup, address-seed cluster), the
@@ -6039,16 +6602,63 @@ const resolveSameSpanLabelConflicts = (entities) => {
6039
6602
  if (dropped.size === 0) return entities;
6040
6603
  return entities.filter((e) => !dropped.has(e));
6041
6604
  };
6605
+ /**
6606
+ * Trailing typographic punctuation that detectors
6607
+ * occasionally swallow when a capture runs to the end
6608
+ * of a sentence or quoted phrase. Stripped from every
6609
+ * non-literal, non-locked entity. Curated dictionary and
6610
+ * gazetteer entries with punctuation that is clearly part of
6611
+ * the literal (`Hello bank!`, `"Juez y parte"`) keep their
6612
+ * own boundaries. Generated/extended spans from the same
6613
+ * sources still pass through cleanup so dangling punctuation
6614
+ * does not become part of the redaction
6615
+ * (e.g. `Bond Hedge Documentation"` →
6616
+ * `Bond Hedge Documentation`).
6617
+ *
6618
+ * `)` is deliberately omitted — monetary amounts are
6619
+ * extended to include trailing "(slovy ...)" / "(in
6620
+ * words ...)" parentheticals where the closing paren
6621
+ * is structural, and stripping it would leave the open
6622
+ * paren dangling. `.` is also omitted because the
6623
+ * trailing-period rule below has label-aware handling
6624
+ * (legal-form abbreviations keep their dot).
6625
+ */
6626
+ const TRAILING_PUNCT_CLASS = `["“”‘’'»!?]`;
6627
+ /**
6628
+ * Leading typographic punctuation that detectors
6629
+ * occasionally swallow when a capture starts at an
6630
+ * opening quote. `(` is deliberately omitted — it is
6631
+ * almost always the opening of a structural
6632
+ * parenthetical (registration number group, monetary
6633
+ * "(slovy ...)" extension) that the detector
6634
+ * intentionally captured.
6635
+ */
6636
+ const LEADING_PUNCT_CLASS = `["“”‘’'«¿¡]`;
6637
+ const STRIP_BY_LABEL = {
6638
+ colon: /[\s,;]+/,
6639
+ default: /[\s:,;]+/
6640
+ };
6641
+ const LEADING_TRIM_BY_LABEL = {
6642
+ colon: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.colon.source}|${LEADING_PUNCT_CLASS})+`),
6643
+ default: new RegExp(`^(?:\\.\\s|${STRIP_BY_LABEL.default.source}|${LEADING_PUNCT_CLASS})+`)
6644
+ };
6645
+ const TRAILING_TRIM_BY_LABEL = {
6646
+ colon: new RegExp(`(?:${STRIP_BY_LABEL.colon.source}|${TRAILING_PUNCT_CLASS})+$`),
6647
+ default: new RegExp(`(?:${STRIP_BY_LABEL.default.source}|${TRAILING_PUNCT_CLASS})+$`)
6648
+ };
6042
6649
  /** Strip leading/trailing whitespace and punctuation. */
6043
6650
  const sanitizeEntities = (entities) => entities.flatMap((e) => {
6044
- if (hasLockedBoundary(e)) return [e];
6045
- const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
6046
- const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
6651
+ if (hasLockedBoundary(e) || hasCuratedLiteralBoundary(e)) return [e];
6652
+ const stripKind = COLON_LABELS.has(e.label) ? "colon" : "default";
6653
+ const leadRe = LEADING_TRIM_BY_LABEL[stripKind];
6654
+ const trailRe = TRAILING_TRIM_BY_LABEL[stripKind];
6655
+ const leadTrimmed = e.text.replace(leadRe, "");
6047
6656
  const lead = e.text.length - leadTrimmed.length;
6048
- let cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
6049
- if (e.label === "organization" && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
6050
- if (!getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix))) cleaned = cleaned.slice(0, -1).trimEnd();
6657
+ let cleaned = leadTrimmed.replace(trailRe, "");
6658
+ if (PERIOD_STRIPPED_LABELS.has(e.label) && cleaned.endsWith(".") && !LITERAL_SOURCES.has(e.source)) {
6659
+ if (!(getKnownLegalSuffixes().some((suffix) => cleaned.endsWith(suffix)) || e.label === "address" && hasKnownAddressFinalAbbrev(cleaned) || e.label === "location" && hasLocationFinalAbbrev(cleaned))) cleaned = cleaned.slice(0, -1).trimEnd();
6051
6660
  }
6661
+ cleaned = cleaned.replace(trailRe, "");
6052
6662
  if (cleaned.length === 0) return [];
6053
6663
  if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
6054
6664
  const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
@@ -6105,7 +6715,7 @@ let amountWordsLoaded = false;
6105
6715
  const getAmountWordsRe = async () => {
6106
6716
  if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
6107
6717
  try {
6108
- const alt = (await import("./amount-words.mjs")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
6718
+ const alt = (await import("./amount-words.mjs").then((n) => n.n)).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
6109
6719
  amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
6110
6720
  } catch {
6111
6721
  amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;