@stll/anonymize-wasm 1.4.9 → 1.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/wasm.mjs CHANGED
@@ -7959,6 +7959,308 @@ const processCountryMatches = (allMatches, sliceStart, sliceEnd, fullText, data)
7959
7959
  return results;
7960
7960
  };
7961
7961
  //#endregion
7962
+ //#region src/config/titles.ts
7963
+ /**
7964
+ * Academic and professional title prefixes.
7965
+ * Plain text; the detector auto-escapes for regex.
7966
+ * Sorted longest-first at build time.
7967
+ */
7968
+ const TITLE_PREFIXES = [
7969
+ "Ing.",
7970
+ "Mgr.",
7971
+ "MgA.",
7972
+ "Bc.",
7973
+ "BcA.",
7974
+ "JUDr.",
7975
+ "MUDr.",
7976
+ "MVDr.",
7977
+ "MDDr.",
7978
+ "PhDr.",
7979
+ "RNDr.",
7980
+ "PaedDr.",
7981
+ "ThDr.",
7982
+ "ThLic.",
7983
+ "ICDr.",
7984
+ "RSDr.",
7985
+ "PharmDr.",
7986
+ "artD.",
7987
+ "akad.",
7988
+ "doc.",
7989
+ "prof.",
7990
+ "ao. Univ.-Prof.",
7991
+ "o. Univ.-Prof.",
7992
+ "Univ.-Prof.",
7993
+ "Hon.-Prof.",
7994
+ "em. Prof.",
7995
+ "Dr. med. dent.",
7996
+ "Dr. med. vet.",
7997
+ "Dr. med.",
7998
+ "Dr. rer. nat.",
7999
+ "Dr. rer. soc.",
8000
+ "Dr. rer. pol.",
8001
+ "Dr. sc. tech.",
8002
+ "Dr. sc. nat.",
8003
+ "Dr. sc. hum.",
8004
+ "Dr. iur.",
8005
+ "Dr. jur.",
8006
+ "Dr. theol.",
8007
+ "Dr. oec.",
8008
+ "Dr. techn.",
8009
+ "Dr. h. c.",
8010
+ "Dr. phil.",
8011
+ "Dr.-Ing.",
8012
+ "Dr. Ing.",
8013
+ "Dr.",
8014
+ "Dipl.-Wirt.-Ing.",
8015
+ "Dipl.-Betriebsw.",
8016
+ "Dipl.-Inform.",
8017
+ "Dipl.-Volksw.",
8018
+ "Dipl.-Psych.",
8019
+ "Dipl.-Phys.",
8020
+ "Dipl.-Chem.",
8021
+ "Dipl.-Biol.",
8022
+ "Dipl.-Math.",
8023
+ "Dipl.-Päd.",
8024
+ "Dipl.-Soz.",
8025
+ "Dipl.-Kfm.",
8026
+ "Dipl.-Jur.",
8027
+ "Dipl. Ing.",
8028
+ "Dipl.-Ing.",
8029
+ "Mag. rer. soc. oec.",
8030
+ "Mag. rer. nat.",
8031
+ "Mag. phil.",
8032
+ "Mag. iur.",
8033
+ "Mag. arch.",
8034
+ "Mag. pharm.",
8035
+ "Mag. (FH)",
8036
+ "Mag.",
8037
+ "Bakk. rer. nat.",
8038
+ "Bakk. techn.",
8039
+ "Bakk. phil.",
8040
+ "Bakk.",
8041
+ "Lic. phil.",
8042
+ "Lic. iur.",
8043
+ "Lic. oec.",
8044
+ "Lic. theol.",
8045
+ "Lic.",
8046
+ "Priv.-Doz.",
8047
+ "PD",
8048
+ "RA"
8049
+ ];
8050
+ /**
8051
+ * Courtesy/honorific titles that precede a person's
8052
+ * name. Sorted alphabetically. The detector escapes
8053
+ * dots and adds \b for entries in HONORIFIC_BOUNDARY.
8054
+ */
8055
+ const HONORIFICS = [
8056
+ "Avv.",
8057
+ "Dame",
8058
+ "Doamna",
8059
+ "Domnul",
8060
+ "Don",
8061
+ "Doña",
8062
+ "Dott.",
8063
+ "Judge",
8064
+ "Justice",
8065
+ "Lady",
8066
+ "Lord",
8067
+ "M.",
8068
+ "Madame",
8069
+ "Mademoiselle",
8070
+ "Maître",
8071
+ "Me",
8072
+ "Messrs",
8073
+ "Miss",
8074
+ "Mlle",
8075
+ "Mme",
8076
+ "Monsieur",
8077
+ "Mr",
8078
+ "Mrs",
8079
+ "Ms",
8080
+ "Pr",
8081
+ "Pr.",
8082
+ "President",
8083
+ "Señor",
8084
+ "Señora",
8085
+ "Sig.",
8086
+ "Sig.ra",
8087
+ "Signor",
8088
+ "Signora",
8089
+ "Signorina",
8090
+ "Sir",
8091
+ "Sr.",
8092
+ "Sra."
8093
+ ];
8094
+ /**
8095
+ * Honorifics that need \b word-boundary anchors
8096
+ * (short or common words that could match mid-word).
8097
+ */
8098
+ const HONORIFIC_BOUNDARY = new Set([
8099
+ "Don",
8100
+ "Doña",
8101
+ "M.",
8102
+ "Me",
8103
+ "Pr",
8104
+ "Pr.",
8105
+ "Señor",
8106
+ "Señora"
8107
+ ]);
8108
+ /**
8109
+ * Honorifics that are abbreviations: a dot after them is an
8110
+ * abbreviation dot (same sentence), so the detector keeps an
8111
+ * optional `.` between the title and the name ("Mr. Smith").
8112
+ * Every other honorific is a full word; a trailing dot ends a
8113
+ * sentence, so the detector must NOT consume it (otherwise a span
8114
+ * like "President. The Employee" crosses the sentence boundary).
8115
+ *
8116
+ * Maintained explicitly, NOT derived from "ends in a dot": "Mr" is
8117
+ * an abbreviation written without a dot, and "Lord" is a full word.
8118
+ */
8119
+ const HONORIFIC_ABBREVIATION = new Set([
8120
+ "Avv.",
8121
+ "Dott.",
8122
+ "M.",
8123
+ "Me",
8124
+ "Messrs",
8125
+ "Mlle",
8126
+ "Mme",
8127
+ "Mr",
8128
+ "Mrs",
8129
+ "Ms",
8130
+ "Pr",
8131
+ "Pr.",
8132
+ "Sig.",
8133
+ "Sig.ra",
8134
+ "Sr.",
8135
+ "Sra."
8136
+ ]);
8137
+ /**
8138
+ * Post-nominal degrees (comma or space separated after name).
8139
+ * Plain text; the detector auto-escapes for regex.
8140
+ */
8141
+ const POST_NOMINALS = [
8142
+ "Ph.D.",
8143
+ "Ph.D",
8144
+ "CSc.",
8145
+ "DrSc.",
8146
+ "ArtD.",
8147
+ "D.Phil.",
8148
+ "DPhil.",
8149
+ "MPhil.",
8150
+ "MBA",
8151
+ "MPA",
8152
+ "LL.M.",
8153
+ "LL.B.",
8154
+ "M.Sc.",
8155
+ "B.Sc.",
8156
+ "MSc.",
8157
+ "BSc.",
8158
+ "M.Eng.",
8159
+ "B.Eng.",
8160
+ "M.A.",
8161
+ "B.A.",
8162
+ "JCD",
8163
+ "JD",
8164
+ "DiS.",
8165
+ "ACCA",
8166
+ "FCCA",
8167
+ "CIPM",
8168
+ "CIPT",
8169
+ "CIPP/E",
8170
+ "CIPP",
8171
+ "KC",
8172
+ "QC"
8173
+ ];
8174
+ const NONWESTERN_HONORIFICS = {
8175
+ in: [
8176
+ "Sri",
8177
+ "Shri",
8178
+ "Smt",
8179
+ "Smt.",
8180
+ "Kumari",
8181
+ "Pandit",
8182
+ "Pt.",
8183
+ "Adv.",
8184
+ "Adv",
8185
+ "Justice",
8186
+ "Hon'ble"
8187
+ ],
8188
+ ar: [
8189
+ "Al-",
8190
+ "El-",
8191
+ "Sheikh",
8192
+ "Shaikh",
8193
+ "Sheikha",
8194
+ "Ustaz",
8195
+ "Ustaza",
8196
+ "Abu",
8197
+ "Umm"
8198
+ ],
8199
+ "zh-Latn": [
8200
+ "Encik",
8201
+ "Puan",
8202
+ "Datuk",
8203
+ "Dato'",
8204
+ "Dato",
8205
+ "Tan Sri",
8206
+ "Tun"
8207
+ ],
8208
+ "ja-Latn": [
8209
+ "-san",
8210
+ "-sama",
8211
+ "-sensei",
8212
+ "Mr.",
8213
+ "Ms."
8214
+ ],
8215
+ ko: ["Sunsaeng", "Gyosu"],
8216
+ th: [
8217
+ "Khun",
8218
+ "Nai",
8219
+ "Nang",
8220
+ "Khunying",
8221
+ "Luang",
8222
+ "Phra",
8223
+ "Mom",
8224
+ "Momluang",
8225
+ "Mommuen",
8226
+ "Momratchawong"
8227
+ ],
8228
+ vi: [
8229
+ "Ông",
8230
+ "Ong",
8231
+ "Ba",
8232
+ "Co"
8233
+ ],
8234
+ fil: [
8235
+ "Atty.",
8236
+ "Atty",
8237
+ "Ginoo",
8238
+ "Ginang",
8239
+ "Binibini",
8240
+ "G.",
8241
+ "Gng.",
8242
+ "Bb."
8243
+ ],
8244
+ id: [
8245
+ "Bapak",
8246
+ "Ibu",
8247
+ "Pak",
8248
+ "Bu",
8249
+ "Raden",
8250
+ "Tuan",
8251
+ "Nyonya",
8252
+ "Nona",
8253
+ "Haji",
8254
+ "Hajjah",
8255
+ "H.",
8256
+ "Hj.",
8257
+ "S.H.",
8258
+ "S.H",
8259
+ "Ir.",
8260
+ "Drs."
8261
+ ]
8262
+ };
8263
+ //#endregion
7962
8264
  //#region src/util/text.ts
7963
8265
  /**
7964
8266
  * Shared text utilities for detectors.
@@ -7990,6 +8292,30 @@ const getCorpus = (ctx) => ctx.nameCorpus;
7990
8292
  const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
7991
8293
  const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
7992
8294
  const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
8295
+ const getNameCorpusNonWesternNames = (ctx = defaultContext) => ctx.nameCorpus?.nonWesternNamesList ?? [];
8296
+ const NONWESTERN_LOCALE_KEYS = [
8297
+ "in",
8298
+ "ar",
8299
+ "ja-latn",
8300
+ "ko",
8301
+ "zh-latn",
8302
+ "th",
8303
+ "vi",
8304
+ "fil",
8305
+ "id"
8306
+ ];
8307
+ const normalizeCorpusLanguage = (language) => language.toLowerCase();
8308
+ const getScopedNonWesternLocaleKeys = (languages) => {
8309
+ if (languages === void 0) return NONWESTERN_LOCALE_KEYS;
8310
+ const allowed = new Set(languages.map(normalizeCorpusLanguage));
8311
+ return NONWESTERN_LOCALE_KEYS.filter((locale) => allowed.has(locale));
8312
+ };
8313
+ const getScopedNonWesternHonorifics = (languages) => {
8314
+ const entries = Object.entries(NONWESTERN_HONORIFICS);
8315
+ if (languages === void 0) return entries.flatMap(([, forms]) => forms);
8316
+ const allowed = new Set(languages.map(normalizeCorpusLanguage));
8317
+ return entries.filter(([locale]) => allowed.has(normalizeCorpusLanguage(locale))).flatMap(([, forms]) => forms);
8318
+ };
7993
8319
  /**
7994
8320
  * Load name corpus data from injected dictionaries
7995
8321
  * and legacy config files. Merges all sources.
@@ -8040,17 +8366,42 @@ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
8040
8366
  const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
8041
8367
  const dedupFirst = dedup(firstNames);
8042
8368
  const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
8043
- const titles = titleMod.default.tokens;
8369
+ const titles = [...titleMod.default.tokens];
8370
+ const scopedNonWesternHonorifics = getScopedNonWesternHonorifics(languages);
8371
+ for (const form of scopedNonWesternHonorifics) titles.push(form.replace(/[.-]+$/u, "").toLowerCase());
8372
+ const dedupTitles = dedup(titles);
8373
+ const titleAbbrSet = /* @__PURE__ */ new Set();
8374
+ for (const prefix of TITLE_PREFIXES) if (prefix.endsWith(".")) titleAbbrSet.add(prefix.replace(/[.-]+$/u, "").toLowerCase());
8375
+ for (const form of HONORIFIC_ABBREVIATION) titleAbbrSet.add(form.replace(/[.-]+$/u, "").toLowerCase());
8376
+ for (const form of scopedNonWesternHonorifics) if (form.endsWith(".")) titleAbbrSet.add(form.replace(/[.-]+$/u, "").toLowerCase());
8044
8377
  const exclusions = exclusionMod.default.words;
8378
+ const nwLocaleKeys = getScopedNonWesternLocaleKeys(languages);
8379
+ const [nwNameMods, nwExcludedMod] = await Promise.all([Promise.all(nwLocaleKeys.map((locale) => import(`../data/names-nw-${locale}.json`))), import("./names-nw-excluded-allcaps.mjs")]);
8380
+ const nonWesternNames = [];
8381
+ for (const mod of nwNameMods) for (const name of mod.default.names) nonWesternNames.push(name);
8382
+ if (dictionaries?.nonWesternNames) {
8383
+ const entries = languages === void 0 ? Object.entries(dictionaries.nonWesternNames) : (() => {
8384
+ const allowed = new Set(languages.map(normalizeCorpusLanguage));
8385
+ return Object.entries(dictionaries.nonWesternNames).filter(([language]) => allowed.has(normalizeCorpusLanguage(language)));
8386
+ })();
8387
+ for (const [, names] of entries) for (const name of names) nonWesternNames.push(name);
8388
+ }
8389
+ const dedupNonWestern = dedup(nonWesternNames);
8390
+ const dedupExcludedAllCaps = dedup(nwExcludedMod.default.words);
8045
8391
  ctx.nameCorpus = {
8046
8392
  firstNames: Object.freeze(new Set(dedupFirst)),
8047
8393
  surnames: Object.freeze(new Set(dedupSurnames)),
8048
- titleTokens: Object.freeze(new Set(titles)),
8394
+ titleTokens: Object.freeze(new Set(dedupTitles)),
8395
+ titleAbbreviations: Object.freeze(titleAbbrSet),
8049
8396
  excludedWords: Object.freeze(new Set(exclusions)),
8397
+ nonWesternNames: Object.freeze(new Set(dedupNonWestern)),
8398
+ excludedAllCaps: Object.freeze(new Set(dedupExcludedAllCaps)),
8050
8399
  firstNamesList: Object.freeze(dedupFirst),
8051
8400
  surnamesList: Object.freeze(dedupSurnames),
8052
- titlesList: Object.freeze(titles),
8053
- excludedList: Object.freeze(exclusions)
8401
+ titlesList: Object.freeze(dedupTitles),
8402
+ excludedList: Object.freeze(exclusions),
8403
+ nonWesternNamesList: Object.freeze(dedupNonWestern),
8404
+ excludedAllCapsList: Object.freeze(dedupExcludedAllCaps)
8054
8405
  };
8055
8406
  } catch (err) {
8056
8407
  ctx.nameCorpusPromise = null;
@@ -8278,6 +8629,8 @@ const TOKEN_TYPE = {
8278
8629
  SURNAME: "surname",
8279
8630
  TITLE: "title",
8280
8631
  ABBREVIATION: "abbreviation",
8632
+ JA_SUFFIX: "ja_suffix",
8633
+ ARABIC_CONNECTOR: "arabic_connector",
8281
8634
  CAPITALIZED: "capitalized",
8282
8635
  OTHER: "other"
8283
8636
  };
@@ -8304,6 +8657,141 @@ const isSurnameToken = (token, corpus) => {
8304
8657
  * abbreviation: "J.", "M.", etc.
8305
8658
  */
8306
8659
  const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
8660
+ /**
8661
+ * Title-case a token, re-capitalizing after apostrophes
8662
+ * so D'Souza, O'Brien, Hon'ble match the corpus.
8663
+ */
8664
+ const titleCaseWithApostrophe = (text) => (text[0]?.toUpperCase() ?? "") + text.slice(1).toLowerCase().replace(/'\p{Ll}/gu, (m) => m.toUpperCase());
8665
+ /**
8666
+ * Check if a token is in the non-Western name corpus.
8667
+ * Tries both title-cased (with apostrophe re-cap) and
8668
+ * raw forms.
8669
+ */
8670
+ const isNonWesternNameToken = (token, corpus) => {
8671
+ if (corpus.nonWesternNames.has(token)) return true;
8672
+ return corpus.nonWesternNames.has(titleCaseWithApostrophe(token));
8673
+ };
8674
+ /** Japanese honorific suffixes attached via hyphen. */
8675
+ const JA_SUFFIXES = new Set([
8676
+ "san",
8677
+ "sama",
8678
+ "sensei"
8679
+ ]);
8680
+ /** Arabic patronymic connectors. */
8681
+ const ARABIC_CONNECTORS = new Set([
8682
+ "bin",
8683
+ "bint",
8684
+ "ibn",
8685
+ "al",
8686
+ "el"
8687
+ ]);
8688
+ /** Al-/El- prefixed name pattern (e.g., Al-Rashid, El-Amin). */
8689
+ const AL_EL_PREFIX_RE = /^[Aa]l-[A-Z][a-z]+$|^[Ee]l-[A-Z][a-z]+$/u;
8690
+ /** CJK name detection: 2-4 Han chars not adjacent to others. */
8691
+ const CJK_NAME_RE = /(?<!\p{Script=Han})\p{Script=Han}{2,4}(?!\p{Script=Han})/gu;
8692
+ /** Han character threshold for CJK-majority documents. */
8693
+ const CJK_HAN_RATIO = .15;
8694
+ const CJK_NON_PERSON_TERMS = new Set([
8695
+ "中国",
8696
+ "中國",
8697
+ "中文",
8698
+ "人民",
8699
+ "公司",
8700
+ "香港",
8701
+ "台湾",
8702
+ "臺灣",
8703
+ "日本",
8704
+ "韩国",
8705
+ "韓國"
8706
+ ]);
8707
+ const CJK_SURNAME_CHARS = new Set([
8708
+ "王",
8709
+ "李",
8710
+ "张",
8711
+ "張",
8712
+ "刘",
8713
+ "劉",
8714
+ "陈",
8715
+ "陳",
8716
+ "杨",
8717
+ "楊",
8718
+ "黄",
8719
+ "黃",
8720
+ "赵",
8721
+ "趙",
8722
+ "吴",
8723
+ "吳",
8724
+ "周",
8725
+ "徐",
8726
+ "孙",
8727
+ "孫",
8728
+ "马",
8729
+ "馬",
8730
+ "朱",
8731
+ "胡",
8732
+ "郭",
8733
+ "何",
8734
+ "林",
8735
+ "高",
8736
+ "梁",
8737
+ "郑",
8738
+ "鄭",
8739
+ "罗",
8740
+ "羅",
8741
+ "宋",
8742
+ "谢",
8743
+ "謝",
8744
+ "唐",
8745
+ "韩",
8746
+ "韓",
8747
+ "曹",
8748
+ "许",
8749
+ "許",
8750
+ "邓",
8751
+ "鄧",
8752
+ "萧",
8753
+ "蕭",
8754
+ "田",
8755
+ "山",
8756
+ "佐",
8757
+ "鈴",
8758
+ "渡",
8759
+ "伊",
8760
+ "中",
8761
+ "小",
8762
+ "吉",
8763
+ "金",
8764
+ "朴",
8765
+ "박",
8766
+ "김",
8767
+ "이",
8768
+ "최",
8769
+ "정",
8770
+ "강",
8771
+ "조",
8772
+ "윤",
8773
+ "장",
8774
+ "임",
8775
+ "한"
8776
+ ].join(""));
8777
+ const isLikelyCjkPersonName = (text) => {
8778
+ if (CJK_NON_PERSON_TERMS.has(text)) return false;
8779
+ const first = text.at(0);
8780
+ return first !== void 0 && CJK_SURNAME_CHARS.has(first);
8781
+ };
8782
+ /** Organization keyword filter. */
8783
+ const ORG_WORDS = /\b(?:Group|Company|LLC|LLP|LP|Inc|Ltd|Corp|Corporation|Holdings|Partners|Association|University|Bank|Fund|Trust|Agency|Government|Ministry|Office|Department|Council|Board|Committee|Commission|Services|Solutions|Technologies|Systems|Analytics|Software)\b/i;
8784
+ const isOrganization = (text) => ORG_WORDS.test(text);
8785
+ /** Deduplicate overlapping entity spans (first wins). */
8786
+ const deduplicateSpans$1 = (entities) => {
8787
+ const sorted = [...entities].sort((a, b) => a.start - b.start || b.end - a.end);
8788
+ const result = [];
8789
+ for (const entity of sorted) {
8790
+ const last = result.at(-1);
8791
+ if (!last || entity.start >= last.end) result.push(entity);
8792
+ }
8793
+ return result;
8794
+ };
8307
8795
  const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
8308
8796
  /**
8309
8797
  * Split text into word segments using Intl.Segmenter.
@@ -8352,21 +8840,55 @@ const classifyToken = (word, corpus, fullText) => {
8352
8840
  text,
8353
8841
  type: TOKEN_TYPE.TITLE,
8354
8842
  start,
8843
+ end,
8844
+ ...corpus.titleAbbreviations.has(stripped) ? { titleAbbreviation: true } : {}
8845
+ };
8846
+ if (JA_SUFFIXES.has(lower)) return {
8847
+ text,
8848
+ type: TOKEN_TYPE.JA_SUFFIX,
8849
+ start,
8355
8850
  end
8356
8851
  };
8852
+ if (ARABIC_CONNECTORS.has(lower)) return {
8853
+ text,
8854
+ type: TOKEN_TYPE.ARABIC_CONNECTOR,
8855
+ start,
8856
+ end
8857
+ };
8858
+ if (AL_EL_PREFIX_RE.test(text)) return {
8859
+ text,
8860
+ type: TOKEN_TYPE.NAME,
8861
+ start,
8862
+ end,
8863
+ nonWestern: true
8864
+ };
8357
8865
  if (isAbbreviation(text)) return {
8358
8866
  text,
8359
8867
  type: TOKEN_TYPE.ABBREVIATION,
8360
8868
  start,
8361
8869
  end
8362
8870
  };
8871
+ if (/^(?:\p{Lu}\.)+\p{Lu}?$/u.test(text)) return {
8872
+ text,
8873
+ type: TOKEN_TYPE.ABBREVIATION,
8874
+ start,
8875
+ end
8876
+ };
8363
8877
  if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
8364
8878
  const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
8365
8879
  const before = fullText.slice(lineStart, start).trimEnd();
8366
8880
  const lastWord = /\p{L}[\p{L}\p{M}'-]*$/u.exec(before)?.[0];
8367
- if (lastWord) {
8368
- const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus);
8369
- if (lookup(lastWord)) return {
8881
+ const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus) || isNonWesternNameToken(token, corpus);
8882
+ if (lastWord && lookup(lastWord)) return {
8883
+ text,
8884
+ type: TOKEN_TYPE.ABBREVIATION,
8885
+ start,
8886
+ end
8887
+ };
8888
+ const afterDot = fullText.slice(end + 1).trimStart();
8889
+ const nextWord = /^\p{L}[\p{L}\p{M}'-]*/u.exec(afterDot)?.[0];
8890
+ if (nextWord) {
8891
+ if (lookup(nextWord) || nextWord.length === 1 && UPPER_START_RE.test(nextWord)) return {
8370
8892
  text,
8371
8893
  type: TOKEN_TYPE.ABBREVIATION,
8372
8894
  start,
@@ -8386,26 +8908,55 @@ const classifyToken = (word, corpus, fullText) => {
8386
8908
  start,
8387
8909
  end
8388
8910
  };
8389
- if (text.length < 3) return {
8911
+ if (text.length < 2) return {
8912
+ text,
8913
+ type: TOKEN_TYPE.OTHER,
8914
+ start,
8915
+ end
8916
+ };
8917
+ if (text.length < 3 && !isNonWesternNameToken(text, corpus) && !JA_SUFFIXES.has(lower) && !ARABIC_CONNECTORS.has(lower) && !(ALL_UPPER_RE.test(text) && !corpus.excludedAllCaps.has(text))) return {
8390
8918
  text,
8391
8919
  type: TOKEN_TYPE.OTHER,
8392
8920
  start,
8393
8921
  end
8394
8922
  };
8395
8923
  if (text.length >= 3 && ALL_UPPER_RE.test(text)) {
8924
+ if (corpus.excludedAllCaps.has(text)) return {
8925
+ text,
8926
+ type: TOKEN_TYPE.OTHER,
8927
+ start,
8928
+ end
8929
+ };
8930
+ const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
8931
+ const nwMatch = isNonWesternNameToken(titleCased, corpus);
8932
+ if (nwMatch && !isFirstNameToken(titleCased, corpus)) return {
8933
+ text,
8934
+ type: TOKEN_TYPE.NAME,
8935
+ start,
8936
+ end,
8937
+ nonWestern: true
8938
+ };
8396
8939
  if (isAllCapsContextLine(fullText, start) && isAllCapsLineNameShaped(fullText, start)) {
8397
- const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
8398
8940
  if (isFirstNameToken(titleCased, corpus)) return {
8399
8941
  text,
8400
8942
  type: TOKEN_TYPE.NAME,
8401
8943
  start,
8402
- end
8944
+ end,
8945
+ ...nwMatch ? { nonWestern: true } : {}
8403
8946
  };
8404
8947
  if (isSurnameToken(titleCased, corpus)) return {
8405
8948
  text,
8406
- type: TOKEN_TYPE.SURNAME,
8949
+ type: TOKEN_TYPE.SURNAME,
8950
+ start,
8951
+ end,
8952
+ ...nwMatch ? { nonWestern: true } : {}
8953
+ };
8954
+ if (nwMatch) return {
8955
+ text,
8956
+ type: TOKEN_TYPE.NAME,
8407
8957
  start,
8408
- end
8958
+ end,
8959
+ nonWestern: true
8409
8960
  };
8410
8961
  }
8411
8962
  return {
@@ -8421,17 +8972,32 @@ const classifyToken = (word, corpus, fullText) => {
8421
8972
  start,
8422
8973
  end
8423
8974
  };
8424
- if (isFirstNameToken(text, corpus)) return {
8975
+ if (isFirstNameToken(text, corpus)) {
8976
+ const nw = isNonWesternNameToken(text, corpus);
8977
+ return {
8978
+ text,
8979
+ type: TOKEN_TYPE.NAME,
8980
+ start,
8981
+ end,
8982
+ ...nw ? { nonWestern: true } : {}
8983
+ };
8984
+ }
8985
+ if (isSurnameToken(text, corpus)) {
8986
+ const nw = isNonWesternNameToken(text, corpus);
8987
+ return {
8988
+ text,
8989
+ type: TOKEN_TYPE.SURNAME,
8990
+ start,
8991
+ end,
8992
+ ...nw ? { nonWestern: true } : {}
8993
+ };
8994
+ }
8995
+ if (isNonWesternNameToken(text, corpus)) return {
8425
8996
  text,
8426
8997
  type: TOKEN_TYPE.NAME,
8427
8998
  start,
8428
- end
8429
- };
8430
- if (isSurnameToken(text, corpus)) return {
8431
- text,
8432
- type: TOKEN_TYPE.SURNAME,
8433
- start,
8434
- end
8999
+ end,
9000
+ nonWestern: true
8435
9001
  };
8436
9002
  return {
8437
9003
  text,
@@ -8443,11 +9009,12 @@ const classifyToken = (word, corpus, fullText) => {
8443
9009
  /**
8444
9010
  * Detect person names by looking up tokens against the
8445
9011
  * name corpus, then chaining adjacent name-like tokens.
9012
+ * Handles both Western and non-Western name patterns.
8446
9013
  *
8447
9014
  * Requires initNameCorpus() to have been called first.
8448
9015
  * If not initialized, returns an empty array.
8449
9016
  *
8450
- * Scoring:
9017
+ * Scoring (Western):
8451
9018
  * TITLE + NAME/SURNAME → 0.95
8452
9019
  * NAME + NAME/SURNAME → 0.9
8453
9020
  * SURNAME + NAME/SURNAME → 0.9
@@ -8455,18 +9022,63 @@ const classifyToken = (word, corpus, fullText) => {
8455
9022
  * ABBREVIATION + NAME → 0.7
8456
9023
  * Standalone NAME → 0.5 (low confidence)
8457
9024
  * Standalone SURNAME → skip (too ambiguous)
8458
- */
8459
- const detectNameCorpus = (fullText, ctx = defaultContext) => {
9025
+ *
9026
+ * Scoring (non-Western, when chain contains nonWestern tokens):
9027
+ * TITLE + (nonWestern|CAPITALIZED) → 0.95
9028
+ * JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
9029
+ * ARABIC_CONNECTOR + nonWestern → 0.9
9030
+ * 2+ nonWestern tokens → 0.9
9031
+ * nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
9032
+ * Standalone nonWestern mid-sentence → 0.5
9033
+ */
9034
+ const detectNameCorpus = (fullText, ctx = defaultContext, options = {}) => {
8460
9035
  const corpus = getCorpus(ctx);
8461
9036
  if (!corpus) return [];
8462
- const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus, fullText));
9037
+ const supplementalMode = options.mode === "supplemental";
8463
9038
  const entities = [];
9039
+ const threshold = Math.ceil(fullText.length * CJK_HAN_RATIO);
9040
+ let hanCount = 0;
9041
+ for (const _ of fullText.matchAll(/\p{Script=Han}/gu)) {
9042
+ hanCount++;
9043
+ if (hanCount >= threshold) break;
9044
+ }
9045
+ if (fullText.length < 100 || hanCount < threshold) {
9046
+ CJK_NAME_RE.lastIndex = 0;
9047
+ let match;
9048
+ while ((match = CJK_NAME_RE.exec(fullText)) !== null) {
9049
+ const cjkText = match[0];
9050
+ if (isLikelyCjkPersonName(cjkText) && !isOrganization(cjkText)) entities.push({
9051
+ start: match.index,
9052
+ end: match.index + cjkText.length,
9053
+ label: "person",
9054
+ text: cjkText,
9055
+ score: .95,
9056
+ source: DETECTION_SOURCES.REGEX
9057
+ });
9058
+ }
9059
+ }
9060
+ const words = segmentWords(fullText);
9061
+ const tokens = [];
9062
+ for (let idx = 0; idx < words.length; idx++) {
9063
+ const word = words[idx];
9064
+ const lower = word.text.toLowerCase();
9065
+ const nextWord = words[idx + 1];
9066
+ if ((lower === "s" || lower === "d" || lower === "w" || lower === "r") && fullText[word.end] === "/" && nextWord && nextWord.start === word.end + 1 && nextWord.text.toLowerCase() === "o") {
9067
+ tokens.push({
9068
+ text: fullText.slice(word.start, word.end + 2),
9069
+ type: TOKEN_TYPE.ARABIC_CONNECTOR,
9070
+ start: word.start,
9071
+ end: word.end + 2
9072
+ });
9073
+ idx++;
9074
+ } else tokens.push(classifyToken(word, corpus, fullText));
9075
+ }
8464
9076
  const consumed = /* @__PURE__ */ new Set();
8465
9077
  for (let i = 0; i < tokens.length; i++) {
8466
9078
  if (consumed.has(i)) continue;
8467
9079
  const token = tokens[i];
8468
9080
  if (!token) continue;
8469
- if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
9081
+ if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION && token.type !== TOKEN_TYPE.ARABIC_CONNECTOR) continue;
8470
9082
  const MAX_CHAIN = 5;
8471
9083
  const chain = [token];
8472
9084
  let j = i + 1;
@@ -8476,10 +9088,12 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
8476
9088
  const prev = chain.at(-1);
8477
9089
  if (prev) {
8478
9090
  const gap = fullText.slice(prev.end, next.start);
8479
- const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap);
9091
+ const periodIsPartOfPrevToken = prev.type === TOKEN_TYPE.ABBREVIATION || prev.type === TOKEN_TYPE.TITLE && prev.titleAbbreviation === true;
9092
+ const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap) && !periodIsPartOfPrevToken;
8480
9093
  if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
9094
+ if (next.type === TOKEN_TYPE.JA_SUFFIX && gap !== "-" && gap.trim() !== "") break;
8481
9095
  }
8482
- if (next.type === TOKEN_TYPE.NAME || next.type === TOKEN_TYPE.SURNAME || next.type === TOKEN_TYPE.TITLE || next.type === TOKEN_TYPE.ABBREVIATION || next.type === TOKEN_TYPE.CAPITALIZED) {
9096
+ if (next.type !== TOKEN_TYPE.OTHER) {
8483
9097
  chain.push(next);
8484
9098
  j++;
8485
9099
  } else break;
@@ -8488,10 +9102,24 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
8488
9102
  const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
8489
9103
  const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
8490
9104
  const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
9105
+ const hasNonWestern = chain.some((t) => t.nonWestern === true);
9106
+ const hasJaSuffix = chain.some((t) => t.type === TOKEN_TYPE.JA_SUFFIX);
9107
+ const hasArabicConnector = chain.some((t) => t.type === TOKEN_TYPE.ARABIC_CONNECTOR);
8491
9108
  const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
8492
9109
  const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
9110
+ const nonWesternCount = chain.filter((t) => t.nonWestern).length;
8493
9111
  let score;
8494
- if (hasTitle && hasCorpusName) score = .95;
9112
+ if (hasNonWestern) {
9113
+ if (hasTitle && (nonWesternCount > 0 || capitalizedCount > 0)) score = .95;
9114
+ else if (hasJaSuffix && (capitalizedCount > 0 || nonWesternCount > 0)) score = .9;
9115
+ else if (hasArabicConnector && nonWesternCount > 0) score = .9;
9116
+ else if (nonWesternCount >= 2) score = .9;
9117
+ else if (nonWesternCount > 0 && (capitalizedCount > 0 || hasAbbreviation)) score = .9;
9118
+ else if (nonWesternCount === 1 && chain.length === 1 && !isSentenceStart(fullText, token.start)) score = .5;
9119
+ else continue;
9120
+ if (supplementalMode && score < .9) continue;
9121
+ } else if (supplementalMode) continue;
9122
+ else if (hasTitle && hasCorpusName) score = .95;
8495
9123
  else if (corpusCount >= 2) score = .9;
8496
9124
  else if (hasCorpusName && capitalizedCount > 0) score = .7;
8497
9125
  else if (hasAbbreviation && hasCorpusName) score = .7;
@@ -8502,7 +9130,10 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
8502
9130
  score = .5;
8503
9131
  } else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
8504
9132
  else if (hasTitle && chain.length === 1) continue;
8505
- else {
9133
+ else if (hasJaSuffix || hasArabicConnector) {
9134
+ if (!hasCorpusName && !hasFirstName) continue;
9135
+ score = .5;
9136
+ } else {
8506
9137
  if (!hasCorpusName) continue;
8507
9138
  score = .5;
8508
9139
  }
@@ -8512,6 +9143,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
8512
9143
  const start = first.start;
8513
9144
  const end = last.end;
8514
9145
  const text = fullText.slice(start, end);
9146
+ if (isOrganization(text)) continue;
8515
9147
  for (let k = i; k < i + chain.length; k++) consumed.add(k);
8516
9148
  entities.push({
8517
9149
  start,
@@ -8522,7 +9154,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
8522
9154
  source: DETECTION_SOURCES.REGEX
8523
9155
  });
8524
9156
  }
8525
- return entities;
9157
+ return deduplicateSpans$1(entities);
8526
9158
  };
8527
9159
  //#endregion
8528
9160
  //#region src/detectors/signatures.ts
@@ -8634,219 +9266,6 @@ const detectSignatures = (fullText, _ctx = defaultContext) => {
8634
9266
  return results;
8635
9267
  };
8636
9268
  //#endregion
8637
- //#region src/config/titles.ts
8638
- /**
8639
- * Academic and professional title prefixes.
8640
- * Plain text; the detector auto-escapes for regex.
8641
- * Sorted longest-first at build time.
8642
- */
8643
- const TITLE_PREFIXES = [
8644
- "Ing.",
8645
- "Mgr.",
8646
- "MgA.",
8647
- "Bc.",
8648
- "BcA.",
8649
- "JUDr.",
8650
- "MUDr.",
8651
- "MVDr.",
8652
- "MDDr.",
8653
- "PhDr.",
8654
- "RNDr.",
8655
- "PaedDr.",
8656
- "ThDr.",
8657
- "ThLic.",
8658
- "ICDr.",
8659
- "RSDr.",
8660
- "PharmDr.",
8661
- "artD.",
8662
- "akad.",
8663
- "doc.",
8664
- "prof.",
8665
- "ao. Univ.-Prof.",
8666
- "o. Univ.-Prof.",
8667
- "Univ.-Prof.",
8668
- "Hon.-Prof.",
8669
- "em. Prof.",
8670
- "Dr. med. dent.",
8671
- "Dr. med. vet.",
8672
- "Dr. med.",
8673
- "Dr. rer. nat.",
8674
- "Dr. rer. soc.",
8675
- "Dr. rer. pol.",
8676
- "Dr. sc. tech.",
8677
- "Dr. sc. nat.",
8678
- "Dr. sc. hum.",
8679
- "Dr. iur.",
8680
- "Dr. jur.",
8681
- "Dr. theol.",
8682
- "Dr. oec.",
8683
- "Dr. techn.",
8684
- "Dr. h. c.",
8685
- "Dr. phil.",
8686
- "Dr.-Ing.",
8687
- "Dr. Ing.",
8688
- "Dr.",
8689
- "Dipl.-Wirt.-Ing.",
8690
- "Dipl.-Betriebsw.",
8691
- "Dipl.-Inform.",
8692
- "Dipl.-Volksw.",
8693
- "Dipl.-Psych.",
8694
- "Dipl.-Phys.",
8695
- "Dipl.-Chem.",
8696
- "Dipl.-Biol.",
8697
- "Dipl.-Math.",
8698
- "Dipl.-Päd.",
8699
- "Dipl.-Soz.",
8700
- "Dipl.-Kfm.",
8701
- "Dipl.-Jur.",
8702
- "Dipl. Ing.",
8703
- "Dipl.-Ing.",
8704
- "Mag. rer. soc. oec.",
8705
- "Mag. rer. nat.",
8706
- "Mag. phil.",
8707
- "Mag. iur.",
8708
- "Mag. arch.",
8709
- "Mag. pharm.",
8710
- "Mag. (FH)",
8711
- "Mag.",
8712
- "Bakk. rer. nat.",
8713
- "Bakk. techn.",
8714
- "Bakk. phil.",
8715
- "Bakk.",
8716
- "Lic. phil.",
8717
- "Lic. iur.",
8718
- "Lic. oec.",
8719
- "Lic. theol.",
8720
- "Lic.",
8721
- "Priv.-Doz.",
8722
- "PD",
8723
- "RA"
8724
- ];
8725
- /**
8726
- * Courtesy/honorific titles that precede a person's
8727
- * name. Sorted alphabetically. The detector escapes
8728
- * dots and adds \b for entries in HONORIFIC_BOUNDARY.
8729
- */
8730
- const HONORIFICS = [
8731
- "Avv.",
8732
- "Dame",
8733
- "Doamna",
8734
- "Domnul",
8735
- "Don",
8736
- "Doña",
8737
- "Dott.",
8738
- "Judge",
8739
- "Justice",
8740
- "Lady",
8741
- "Lord",
8742
- "M.",
8743
- "Madame",
8744
- "Mademoiselle",
8745
- "Maître",
8746
- "Me",
8747
- "Messrs",
8748
- "Miss",
8749
- "Mlle",
8750
- "Mme",
8751
- "Monsieur",
8752
- "Mr",
8753
- "Mrs",
8754
- "Ms",
8755
- "Pr",
8756
- "Pr.",
8757
- "President",
8758
- "Señor",
8759
- "Señora",
8760
- "Sig.",
8761
- "Sig.ra",
8762
- "Signor",
8763
- "Signora",
8764
- "Signorina",
8765
- "Sir",
8766
- "Sr.",
8767
- "Sra."
8768
- ];
8769
- /**
8770
- * Honorifics that need \b word-boundary anchors
8771
- * (short or common words that could match mid-word).
8772
- */
8773
- const HONORIFIC_BOUNDARY = new Set([
8774
- "Don",
8775
- "Doña",
8776
- "M.",
8777
- "Me",
8778
- "Pr",
8779
- "Pr.",
8780
- "Señor",
8781
- "Señora"
8782
- ]);
8783
- /**
8784
- * Honorifics that are abbreviations: a dot after them is an
8785
- * abbreviation dot (same sentence), so the detector keeps an
8786
- * optional `.` between the title and the name ("Mr. Smith").
8787
- * Every other honorific is a full word — a trailing dot ends a
8788
- * sentence, so the detector must NOT consume it (otherwise a span
8789
- * like "President. The Employee" crosses the sentence boundary).
8790
- *
8791
- * Maintained explicitly, NOT derived from "ends in a dot": "Mr" is
8792
- * an abbreviation written without a dot, and "Lord" is a full word.
8793
- */
8794
- const HONORIFIC_ABBREVIATION = new Set([
8795
- "Avv.",
8796
- "Dott.",
8797
- "M.",
8798
- "Me",
8799
- "Messrs",
8800
- "Mlle",
8801
- "Mme",
8802
- "Mr",
8803
- "Mrs",
8804
- "Ms",
8805
- "Pr",
8806
- "Pr.",
8807
- "Sig.",
8808
- "Sig.ra",
8809
- "Sr.",
8810
- "Sra."
8811
- ]);
8812
- /**
8813
- * Post-nominal degrees (comma or space separated after name).
8814
- * Plain text; the detector auto-escapes for regex.
8815
- */
8816
- const POST_NOMINALS = [
8817
- "Ph.D.",
8818
- "Ph.D",
8819
- "CSc.",
8820
- "DrSc.",
8821
- "ArtD.",
8822
- "D.Phil.",
8823
- "DPhil.",
8824
- "MPhil.",
8825
- "MBA",
8826
- "MPA",
8827
- "LL.M.",
8828
- "LL.B.",
8829
- "M.Sc.",
8830
- "B.Sc.",
8831
- "MSc.",
8832
- "BSc.",
8833
- "M.Eng.",
8834
- "B.Eng.",
8835
- "M.A.",
8836
- "B.A.",
8837
- "JCD",
8838
- "JD",
8839
- "DiS.",
8840
- "ACCA",
8841
- "FCCA",
8842
- "CIPM",
8843
- "CIPT",
8844
- "CIPP/E",
8845
- "CIPP",
8846
- "KC",
8847
- "QC"
8848
- ];
8849
- //#endregion
8850
9269
  //#region src/data/amount-words.json
8851
9270
  var amount_words_exports = /* @__PURE__ */ __exportAll({
8852
9271
  _comment: () => _comment$1,
@@ -10721,6 +11140,24 @@ const TRIGGER_LOOKAHEAD_MARGIN = 128;
10721
11140
  const LINE_TRIGGER_LOOKAHEAD = 2048;
10722
11141
  const MATCH_PATTERN_LOOKAHEAD = 512;
10723
11142
  const PHONE_VALUE_START_RE = /^[+(\d]/;
11143
+ /**
11144
+ * Phone-shaped run: an opening "+", "(", or digit, then
11145
+ * any mix of digits, whitespace, parens, dots, slashes,
11146
+ * and dash variants. The dash class includes every
11147
+ * typographic dash (U+2011 non-breaking hyphen, U+2013
11148
+ * en-dash, etc.) via `DASH_INNER` so a Unicode separator
11149
+ * ("+1 555‑123‑4567") does not break the run and leave the
11150
+ * number unredacted.
11151
+ */
11152
+ const PHONE_SHAPE_PREFIX_RE = new RegExp(`^[+(\\d][\\d\\s()./${DASH_INNER}]*`);
11153
+ /**
11154
+ * Optional phone-extension suffix following the numeric
11155
+ * run: "ext", "ext.", "extension", or "x" (case-
11156
+ * insensitive) plus 1-6 digits, with optional leading
11157
+ * whitespace. Matches "+1 555 123 4567 ext. 89" and
11158
+ * "555-1234 x42" so the shape bound keeps the extension.
11159
+ */
11160
+ const PHONE_EXTENSION_SUFFIX_RE = /^\s*(?:ext\.?|extension|x)\s*\d{1,6}/i;
10724
11161
  const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
10725
11162
  const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
10726
11163
  const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
@@ -10813,6 +11250,19 @@ const extractValue = (text, triggerEnd, strategy, label) => {
10813
11250
  end = inlineLabel.index;
10814
11251
  foundLineStop = true;
10815
11252
  }
11253
+ const region = valueText.slice(0, end);
11254
+ const sentenceBreak = /\.\s/.exec(region);
11255
+ const shapeRegion = sentenceBreak !== null ? region.slice(0, sentenceBreak.index) : region;
11256
+ const shape = PHONE_SHAPE_PREFIX_RE.exec(shapeRegion);
11257
+ if (shape) {
11258
+ let shapeEnd = shape[0].replace(/\D+$/, "").length;
11259
+ const extension = PHONE_EXTENSION_SUFFIX_RE.exec(region.slice(shapeEnd));
11260
+ if (extension !== null) shapeEnd += extension[0].length;
11261
+ if (shapeEnd > 0 && shapeEnd < end) {
11262
+ end = shapeEnd > MAX_TRIGGER_VALUE_LEN ? Math.min(capAtWordBoundary(valueText, MAX_TRIGGER_VALUE_LEN), MAX_TRIGGER_VALUE_LEN) : shapeEnd;
11263
+ foundLineStop = true;
11264
+ }
11265
+ }
10816
11266
  }
10817
11267
  if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
10818
11268
  const rawSlice = valueText.slice(0, end);
@@ -10991,10 +11441,20 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
10991
11441
  if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
10992
11442
  const triggerEnd = match.end;
10993
11443
  const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
10994
- const value = rawValue ? stripQuotes(rawValue) : null;
11444
+ let value = rawValue ? stripQuotes(rawValue) : null;
10995
11445
  if (value) {
10996
11446
  if (!applyValidations(value.text, rule.validations)) continue;
10997
11447
  if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
11448
+ if (rule.label === "phone number" && value.text.length > MAX_TRIGGER_VALUE_LEN && fullText[value.end] !== "\n" && fullText[value.end] !== " ") {
11449
+ const cappedEnd = Math.min(capAtWordBoundary(value.text, MAX_TRIGGER_VALUE_LEN), MAX_TRIGGER_VALUE_LEN);
11450
+ const cappedText = value.text.slice(0, cappedEnd).trimEnd();
11451
+ if (cappedText.length === 0) continue;
11452
+ value = {
11453
+ ...value,
11454
+ end: value.start + cappedText.length,
11455
+ text: cappedText
11456
+ };
11457
+ }
10998
11458
  const entityStart = rule.includeTrigger ? match.start : value.start;
10999
11459
  let entityEnd = value.end;
11000
11460
  let entityText = fullText.slice(entityStart, entityEnd);
@@ -11892,6 +12352,7 @@ const patternSources = (sources) => {
11892
12352
  if (sources === void 0) return EMPTY_PATTERN_SOURCES;
11893
12353
  return Array.isArray(sources) ? sources : [sources];
11894
12354
  };
12355
+ const hasPersonNameSource = (match) => match.sources.includes("first-name") || match.sources.includes("surname");
11895
12356
  const addPatternLabel = (list, index, label) => {
11896
12357
  const existing = list[index];
11897
12358
  if (existing === void 0) {
@@ -12238,6 +12699,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
12238
12699
  const first = chain.at(0);
12239
12700
  const last = chain.at(-1);
12240
12701
  if (!first || !last) continue;
12702
+ if (!chain.some(hasPersonNameSource)) continue;
12241
12703
  if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
12242
12704
  const extended = extendPersonName(fullText, first.start, last.end, ctx);
12243
12705
  const score = chain.length >= 2 ? .9 : .5;
@@ -13820,6 +14282,7 @@ const applyHotwordRules = (entities, fullText) => {
13820
14282
  /** Max gap (in chars) between entities to merge. */
13821
14283
  const MAX_GAP = 3;
13822
14284
  const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
14285
+ const hasDetectorLockedBoundary = (entity) => entity.label === "phone number" && entity.source === DETECTION_SOURCES.TRIGGER;
13823
14286
  /**
13824
14287
  * Characters allowed in the gap between two adjacent
13825
14288
  * same-label entities that should be merged: spaces,
@@ -14000,7 +14463,7 @@ const fixPartialWords = (entities, fullText) => {
14000
14463
  })).sort((a, b) => a.entity.end - b.entity.end);
14001
14464
  const endPositions = byEnd.map((x) => x.entity.end);
14002
14465
  return sorted.map((e, eIdx) => {
14003
- if (hasLockedBoundary$1(e)) return e;
14466
+ if (hasLockedBoundary$1(e) || hasDetectorLockedBoundary(e)) return e;
14004
14467
  if (e.text !== fullText.slice(e.start, e.end)) return e;
14005
14468
  let newStart = wordStartAt(e.start, boundaries, fullText);
14006
14469
  let newEnd = wordEndAt(e.end, boundaries, fullText);
@@ -14403,6 +14866,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
14403
14866
  const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
14404
14867
  const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
14405
14868
  const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
14869
+ const isCoveredByDenyListEntity = (entity, denyListEntity) => entity.label === denyListEntity.label && denyListEntity.start <= entity.start && denyListEntity.end >= entity.end;
14870
+ const filterNameCorpusCoveredByDenyList = (nameCorpusEntities, denyListEntities) => nameCorpusEntities.filter((entity) => !denyListEntities.some((denyListEntity) => isCoveredByDenyListEntity(entity, denyListEntity)));
14406
14871
  const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
14407
14872
  const BARE_POSTAL_CODE_RE = /^\s*(?:\d{3}\s?\d{2}|\d{2}[-–]\d{3}|\d{5}(?:[-–]\d{3,4})?)\s*$/u;
14408
14873
  const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
@@ -14923,6 +15388,13 @@ const runPipeline = async (options) => {
14923
15388
  const rawDenyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
14924
15389
  const denyListEntities = filterAllowedLabels(rawDenyListEntities, preHotwordAllowedLabels);
14925
15390
  if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
15391
+ if (config.enableNameCorpus && config.enableDenyList) {
15392
+ await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
15393
+ checkAbort(signal);
15394
+ rawNameCorpusEntities = filterNameCorpusCoveredByDenyList(detectNameCorpus(fullText, ctx, { mode: "supplemental" }), rawDenyListEntities);
15395
+ nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
15396
+ log("name-corpus", `${nameCorpusEntities.length} supplemental matches`);
15397
+ }
14926
15398
  const rawGazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
14927
15399
  const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
14928
15400
  if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
@@ -15618,6 +16090,6 @@ const levenshtein = (rawA, rawB) => {
15618
16090
  //#region src/wasm.ts
15619
16091
  initTextSearch(TextSearch);
15620
16092
  //#endregion
15621
- export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
16093
+ export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
15622
16094
 
15623
16095
  //# sourceMappingURL=wasm.mjs.map