@stll/anonymize-wasm 1.4.9 → 1.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/names-nw-ar.mjs +202 -0
- package/dist/names-nw-ar.mjs.map +1 -0
- package/dist/names-nw-excluded-allcaps.mjs +112 -0
- package/dist/names-nw-excluded-allcaps.mjs.map +1 -0
- package/dist/names-nw-fil.mjs +202 -0
- package/dist/names-nw-fil.mjs.map +1 -0
- package/dist/names-nw-id.mjs +210 -0
- package/dist/names-nw-id.mjs.map +1 -0
- package/dist/names-nw-in.mjs +526 -0
- package/dist/names-nw-in.mjs.map +1 -0
- package/dist/names-nw-ja-latn.mjs +260 -0
- package/dist/names-nw-ja-latn.mjs.map +1 -0
- package/dist/names-nw-ko.mjs +162 -0
- package/dist/names-nw-ko.mjs.map +1 -0
- package/dist/names-nw-th.mjs +188 -0
- package/dist/names-nw-th.mjs.map +1 -0
- package/dist/names-nw-vi.mjs +151 -0
- package/dist/names-nw-vi.mjs.map +1 -0
- package/dist/names-nw-zh-latn.mjs +197 -0
- package/dist/names-nw-zh-latn.mjs.map +1 -0
- package/dist/wasm.d.mts +34 -5
- package/dist/wasm.mjs +729 -246
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -7959,6 +7959,308 @@ const processCountryMatches = (allMatches, sliceStart, sliceEnd, fullText, data)
|
|
|
7959
7959
|
return results;
|
|
7960
7960
|
};
|
|
7961
7961
|
//#endregion
|
|
7962
|
+
//#region src/config/titles.ts
|
|
7963
|
+
/**
|
|
7964
|
+
* Academic and professional title prefixes.
|
|
7965
|
+
* Plain text; the detector auto-escapes for regex.
|
|
7966
|
+
* Sorted longest-first at build time.
|
|
7967
|
+
*/
|
|
7968
|
+
const TITLE_PREFIXES = [
|
|
7969
|
+
"Ing.",
|
|
7970
|
+
"Mgr.",
|
|
7971
|
+
"MgA.",
|
|
7972
|
+
"Bc.",
|
|
7973
|
+
"BcA.",
|
|
7974
|
+
"JUDr.",
|
|
7975
|
+
"MUDr.",
|
|
7976
|
+
"MVDr.",
|
|
7977
|
+
"MDDr.",
|
|
7978
|
+
"PhDr.",
|
|
7979
|
+
"RNDr.",
|
|
7980
|
+
"PaedDr.",
|
|
7981
|
+
"ThDr.",
|
|
7982
|
+
"ThLic.",
|
|
7983
|
+
"ICDr.",
|
|
7984
|
+
"RSDr.",
|
|
7985
|
+
"PharmDr.",
|
|
7986
|
+
"artD.",
|
|
7987
|
+
"akad.",
|
|
7988
|
+
"doc.",
|
|
7989
|
+
"prof.",
|
|
7990
|
+
"ao. Univ.-Prof.",
|
|
7991
|
+
"o. Univ.-Prof.",
|
|
7992
|
+
"Univ.-Prof.",
|
|
7993
|
+
"Hon.-Prof.",
|
|
7994
|
+
"em. Prof.",
|
|
7995
|
+
"Dr. med. dent.",
|
|
7996
|
+
"Dr. med. vet.",
|
|
7997
|
+
"Dr. med.",
|
|
7998
|
+
"Dr. rer. nat.",
|
|
7999
|
+
"Dr. rer. soc.",
|
|
8000
|
+
"Dr. rer. pol.",
|
|
8001
|
+
"Dr. sc. tech.",
|
|
8002
|
+
"Dr. sc. nat.",
|
|
8003
|
+
"Dr. sc. hum.",
|
|
8004
|
+
"Dr. iur.",
|
|
8005
|
+
"Dr. jur.",
|
|
8006
|
+
"Dr. theol.",
|
|
8007
|
+
"Dr. oec.",
|
|
8008
|
+
"Dr. techn.",
|
|
8009
|
+
"Dr. h. c.",
|
|
8010
|
+
"Dr. phil.",
|
|
8011
|
+
"Dr.-Ing.",
|
|
8012
|
+
"Dr. Ing.",
|
|
8013
|
+
"Dr.",
|
|
8014
|
+
"Dipl.-Wirt.-Ing.",
|
|
8015
|
+
"Dipl.-Betriebsw.",
|
|
8016
|
+
"Dipl.-Inform.",
|
|
8017
|
+
"Dipl.-Volksw.",
|
|
8018
|
+
"Dipl.-Psych.",
|
|
8019
|
+
"Dipl.-Phys.",
|
|
8020
|
+
"Dipl.-Chem.",
|
|
8021
|
+
"Dipl.-Biol.",
|
|
8022
|
+
"Dipl.-Math.",
|
|
8023
|
+
"Dipl.-Päd.",
|
|
8024
|
+
"Dipl.-Soz.",
|
|
8025
|
+
"Dipl.-Kfm.",
|
|
8026
|
+
"Dipl.-Jur.",
|
|
8027
|
+
"Dipl. Ing.",
|
|
8028
|
+
"Dipl.-Ing.",
|
|
8029
|
+
"Mag. rer. soc. oec.",
|
|
8030
|
+
"Mag. rer. nat.",
|
|
8031
|
+
"Mag. phil.",
|
|
8032
|
+
"Mag. iur.",
|
|
8033
|
+
"Mag. arch.",
|
|
8034
|
+
"Mag. pharm.",
|
|
8035
|
+
"Mag. (FH)",
|
|
8036
|
+
"Mag.",
|
|
8037
|
+
"Bakk. rer. nat.",
|
|
8038
|
+
"Bakk. techn.",
|
|
8039
|
+
"Bakk. phil.",
|
|
8040
|
+
"Bakk.",
|
|
8041
|
+
"Lic. phil.",
|
|
8042
|
+
"Lic. iur.",
|
|
8043
|
+
"Lic. oec.",
|
|
8044
|
+
"Lic. theol.",
|
|
8045
|
+
"Lic.",
|
|
8046
|
+
"Priv.-Doz.",
|
|
8047
|
+
"PD",
|
|
8048
|
+
"RA"
|
|
8049
|
+
];
|
|
8050
|
+
/**
|
|
8051
|
+
* Courtesy/honorific titles that precede a person's
|
|
8052
|
+
* name. Sorted alphabetically. The detector escapes
|
|
8053
|
+
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
8054
|
+
*/
|
|
8055
|
+
const HONORIFICS = [
|
|
8056
|
+
"Avv.",
|
|
8057
|
+
"Dame",
|
|
8058
|
+
"Doamna",
|
|
8059
|
+
"Domnul",
|
|
8060
|
+
"Don",
|
|
8061
|
+
"Doña",
|
|
8062
|
+
"Dott.",
|
|
8063
|
+
"Judge",
|
|
8064
|
+
"Justice",
|
|
8065
|
+
"Lady",
|
|
8066
|
+
"Lord",
|
|
8067
|
+
"M.",
|
|
8068
|
+
"Madame",
|
|
8069
|
+
"Mademoiselle",
|
|
8070
|
+
"Maître",
|
|
8071
|
+
"Me",
|
|
8072
|
+
"Messrs",
|
|
8073
|
+
"Miss",
|
|
8074
|
+
"Mlle",
|
|
8075
|
+
"Mme",
|
|
8076
|
+
"Monsieur",
|
|
8077
|
+
"Mr",
|
|
8078
|
+
"Mrs",
|
|
8079
|
+
"Ms",
|
|
8080
|
+
"Pr",
|
|
8081
|
+
"Pr.",
|
|
8082
|
+
"President",
|
|
8083
|
+
"Señor",
|
|
8084
|
+
"Señora",
|
|
8085
|
+
"Sig.",
|
|
8086
|
+
"Sig.ra",
|
|
8087
|
+
"Signor",
|
|
8088
|
+
"Signora",
|
|
8089
|
+
"Signorina",
|
|
8090
|
+
"Sir",
|
|
8091
|
+
"Sr.",
|
|
8092
|
+
"Sra."
|
|
8093
|
+
];
|
|
8094
|
+
/**
|
|
8095
|
+
* Honorifics that need \b word-boundary anchors
|
|
8096
|
+
* (short or common words that could match mid-word).
|
|
8097
|
+
*/
|
|
8098
|
+
const HONORIFIC_BOUNDARY = new Set([
|
|
8099
|
+
"Don",
|
|
8100
|
+
"Doña",
|
|
8101
|
+
"M.",
|
|
8102
|
+
"Me",
|
|
8103
|
+
"Pr",
|
|
8104
|
+
"Pr.",
|
|
8105
|
+
"Señor",
|
|
8106
|
+
"Señora"
|
|
8107
|
+
]);
|
|
8108
|
+
/**
|
|
8109
|
+
* Honorifics that are abbreviations: a dot after them is an
|
|
8110
|
+
* abbreviation dot (same sentence), so the detector keeps an
|
|
8111
|
+
* optional `.` between the title and the name ("Mr. Smith").
|
|
8112
|
+
* Every other honorific is a full word; a trailing dot ends a
|
|
8113
|
+
* sentence, so the detector must NOT consume it (otherwise a span
|
|
8114
|
+
* like "President. The Employee" crosses the sentence boundary).
|
|
8115
|
+
*
|
|
8116
|
+
* Maintained explicitly, NOT derived from "ends in a dot": "Mr" is
|
|
8117
|
+
* an abbreviation written without a dot, and "Lord" is a full word.
|
|
8118
|
+
*/
|
|
8119
|
+
const HONORIFIC_ABBREVIATION = new Set([
|
|
8120
|
+
"Avv.",
|
|
8121
|
+
"Dott.",
|
|
8122
|
+
"M.",
|
|
8123
|
+
"Me",
|
|
8124
|
+
"Messrs",
|
|
8125
|
+
"Mlle",
|
|
8126
|
+
"Mme",
|
|
8127
|
+
"Mr",
|
|
8128
|
+
"Mrs",
|
|
8129
|
+
"Ms",
|
|
8130
|
+
"Pr",
|
|
8131
|
+
"Pr.",
|
|
8132
|
+
"Sig.",
|
|
8133
|
+
"Sig.ra",
|
|
8134
|
+
"Sr.",
|
|
8135
|
+
"Sra."
|
|
8136
|
+
]);
|
|
8137
|
+
/**
|
|
8138
|
+
* Post-nominal degrees (comma or space separated after name).
|
|
8139
|
+
* Plain text; the detector auto-escapes for regex.
|
|
8140
|
+
*/
|
|
8141
|
+
const POST_NOMINALS = [
|
|
8142
|
+
"Ph.D.",
|
|
8143
|
+
"Ph.D",
|
|
8144
|
+
"CSc.",
|
|
8145
|
+
"DrSc.",
|
|
8146
|
+
"ArtD.",
|
|
8147
|
+
"D.Phil.",
|
|
8148
|
+
"DPhil.",
|
|
8149
|
+
"MPhil.",
|
|
8150
|
+
"MBA",
|
|
8151
|
+
"MPA",
|
|
8152
|
+
"LL.M.",
|
|
8153
|
+
"LL.B.",
|
|
8154
|
+
"M.Sc.",
|
|
8155
|
+
"B.Sc.",
|
|
8156
|
+
"MSc.",
|
|
8157
|
+
"BSc.",
|
|
8158
|
+
"M.Eng.",
|
|
8159
|
+
"B.Eng.",
|
|
8160
|
+
"M.A.",
|
|
8161
|
+
"B.A.",
|
|
8162
|
+
"JCD",
|
|
8163
|
+
"JD",
|
|
8164
|
+
"DiS.",
|
|
8165
|
+
"ACCA",
|
|
8166
|
+
"FCCA",
|
|
8167
|
+
"CIPM",
|
|
8168
|
+
"CIPT",
|
|
8169
|
+
"CIPP/E",
|
|
8170
|
+
"CIPP",
|
|
8171
|
+
"KC",
|
|
8172
|
+
"QC"
|
|
8173
|
+
];
|
|
8174
|
+
const NONWESTERN_HONORIFICS = {
|
|
8175
|
+
in: [
|
|
8176
|
+
"Sri",
|
|
8177
|
+
"Shri",
|
|
8178
|
+
"Smt",
|
|
8179
|
+
"Smt.",
|
|
8180
|
+
"Kumari",
|
|
8181
|
+
"Pandit",
|
|
8182
|
+
"Pt.",
|
|
8183
|
+
"Adv.",
|
|
8184
|
+
"Adv",
|
|
8185
|
+
"Justice",
|
|
8186
|
+
"Hon'ble"
|
|
8187
|
+
],
|
|
8188
|
+
ar: [
|
|
8189
|
+
"Al-",
|
|
8190
|
+
"El-",
|
|
8191
|
+
"Sheikh",
|
|
8192
|
+
"Shaikh",
|
|
8193
|
+
"Sheikha",
|
|
8194
|
+
"Ustaz",
|
|
8195
|
+
"Ustaza",
|
|
8196
|
+
"Abu",
|
|
8197
|
+
"Umm"
|
|
8198
|
+
],
|
|
8199
|
+
"zh-Latn": [
|
|
8200
|
+
"Encik",
|
|
8201
|
+
"Puan",
|
|
8202
|
+
"Datuk",
|
|
8203
|
+
"Dato'",
|
|
8204
|
+
"Dato",
|
|
8205
|
+
"Tan Sri",
|
|
8206
|
+
"Tun"
|
|
8207
|
+
],
|
|
8208
|
+
"ja-Latn": [
|
|
8209
|
+
"-san",
|
|
8210
|
+
"-sama",
|
|
8211
|
+
"-sensei",
|
|
8212
|
+
"Mr.",
|
|
8213
|
+
"Ms."
|
|
8214
|
+
],
|
|
8215
|
+
ko: ["Sunsaeng", "Gyosu"],
|
|
8216
|
+
th: [
|
|
8217
|
+
"Khun",
|
|
8218
|
+
"Nai",
|
|
8219
|
+
"Nang",
|
|
8220
|
+
"Khunying",
|
|
8221
|
+
"Luang",
|
|
8222
|
+
"Phra",
|
|
8223
|
+
"Mom",
|
|
8224
|
+
"Momluang",
|
|
8225
|
+
"Mommuen",
|
|
8226
|
+
"Momratchawong"
|
|
8227
|
+
],
|
|
8228
|
+
vi: [
|
|
8229
|
+
"Ông",
|
|
8230
|
+
"Ong",
|
|
8231
|
+
"Ba",
|
|
8232
|
+
"Co"
|
|
8233
|
+
],
|
|
8234
|
+
fil: [
|
|
8235
|
+
"Atty.",
|
|
8236
|
+
"Atty",
|
|
8237
|
+
"Ginoo",
|
|
8238
|
+
"Ginang",
|
|
8239
|
+
"Binibini",
|
|
8240
|
+
"G.",
|
|
8241
|
+
"Gng.",
|
|
8242
|
+
"Bb."
|
|
8243
|
+
],
|
|
8244
|
+
id: [
|
|
8245
|
+
"Bapak",
|
|
8246
|
+
"Ibu",
|
|
8247
|
+
"Pak",
|
|
8248
|
+
"Bu",
|
|
8249
|
+
"Raden",
|
|
8250
|
+
"Tuan",
|
|
8251
|
+
"Nyonya",
|
|
8252
|
+
"Nona",
|
|
8253
|
+
"Haji",
|
|
8254
|
+
"Hajjah",
|
|
8255
|
+
"H.",
|
|
8256
|
+
"Hj.",
|
|
8257
|
+
"S.H.",
|
|
8258
|
+
"S.H",
|
|
8259
|
+
"Ir.",
|
|
8260
|
+
"Drs."
|
|
8261
|
+
]
|
|
8262
|
+
};
|
|
8263
|
+
//#endregion
|
|
7962
8264
|
//#region src/util/text.ts
|
|
7963
8265
|
/**
|
|
7964
8266
|
* Shared text utilities for detectors.
|
|
@@ -7990,6 +8292,41 @@ const getCorpus = (ctx) => ctx.nameCorpus;
|
|
|
7990
8292
|
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
7991
8293
|
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
7992
8294
|
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
8295
|
+
const getNameCorpusNonWesternNames = (ctx = defaultContext) => ctx.nameCorpus?.nonWesternNamesList ?? [];
|
|
8296
|
+
const NONWESTERN_LOCALE_KEYS = [
|
|
8297
|
+
"in",
|
|
8298
|
+
"ar",
|
|
8299
|
+
"ja-latn",
|
|
8300
|
+
"ko",
|
|
8301
|
+
"zh-latn",
|
|
8302
|
+
"th",
|
|
8303
|
+
"vi",
|
|
8304
|
+
"fil",
|
|
8305
|
+
"id"
|
|
8306
|
+
];
|
|
8307
|
+
const NONWESTERN_NAME_IMPORTS = {
|
|
8308
|
+
in: () => import("./names-nw-in.mjs"),
|
|
8309
|
+
ar: () => import("./names-nw-ar.mjs"),
|
|
8310
|
+
"ja-latn": () => import("./names-nw-ja-latn.mjs"),
|
|
8311
|
+
ko: () => import("./names-nw-ko.mjs"),
|
|
8312
|
+
"zh-latn": () => import("./names-nw-zh-latn.mjs"),
|
|
8313
|
+
th: () => import("./names-nw-th.mjs"),
|
|
8314
|
+
vi: () => import("./names-nw-vi.mjs"),
|
|
8315
|
+
fil: () => import("./names-nw-fil.mjs"),
|
|
8316
|
+
id: () => import("./names-nw-id.mjs")
|
|
8317
|
+
};
|
|
8318
|
+
const normalizeCorpusLanguage = (language) => language.toLowerCase();
|
|
8319
|
+
const getScopedNonWesternLocaleKeys = (languages) => {
|
|
8320
|
+
if (languages === void 0) return NONWESTERN_LOCALE_KEYS;
|
|
8321
|
+
const allowed = new Set(languages.map(normalizeCorpusLanguage));
|
|
8322
|
+
return NONWESTERN_LOCALE_KEYS.filter((locale) => allowed.has(locale));
|
|
8323
|
+
};
|
|
8324
|
+
const getScopedNonWesternHonorifics = (languages) => {
|
|
8325
|
+
const entries = Object.entries(NONWESTERN_HONORIFICS);
|
|
8326
|
+
if (languages === void 0) return entries.flatMap(([, forms]) => forms);
|
|
8327
|
+
const allowed = new Set(languages.map(normalizeCorpusLanguage));
|
|
8328
|
+
return entries.filter(([locale]) => allowed.has(normalizeCorpusLanguage(locale))).flatMap(([, forms]) => forms);
|
|
8329
|
+
};
|
|
7993
8330
|
/**
|
|
7994
8331
|
* Load name corpus data from injected dictionaries
|
|
7995
8332
|
* and legacy config files. Merges all sources.
|
|
@@ -8040,17 +8377,42 @@ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
|
8040
8377
|
const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
|
|
8041
8378
|
const dedupFirst = dedup(firstNames);
|
|
8042
8379
|
const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
|
|
8043
|
-
const titles = titleMod.default.tokens;
|
|
8380
|
+
const titles = [...titleMod.default.tokens];
|
|
8381
|
+
const scopedNonWesternHonorifics = getScopedNonWesternHonorifics(languages);
|
|
8382
|
+
for (const form of scopedNonWesternHonorifics) titles.push(form.replace(/[.-]+$/u, "").toLowerCase());
|
|
8383
|
+
const dedupTitles = dedup(titles);
|
|
8384
|
+
const titleAbbrSet = /* @__PURE__ */ new Set();
|
|
8385
|
+
for (const prefix of TITLE_PREFIXES) if (prefix.endsWith(".")) titleAbbrSet.add(prefix.replace(/[.-]+$/u, "").toLowerCase());
|
|
8386
|
+
for (const form of HONORIFIC_ABBREVIATION) titleAbbrSet.add(form.replace(/[.-]+$/u, "").toLowerCase());
|
|
8387
|
+
for (const form of scopedNonWesternHonorifics) if (form.endsWith(".")) titleAbbrSet.add(form.replace(/[.-]+$/u, "").toLowerCase());
|
|
8044
8388
|
const exclusions = exclusionMod.default.words;
|
|
8389
|
+
const nwLocaleKeys = getScopedNonWesternLocaleKeys(languages);
|
|
8390
|
+
const [nwNameMods, nwExcludedMod] = await Promise.all([Promise.all(nwLocaleKeys.map((locale) => NONWESTERN_NAME_IMPORTS[locale]())), import("./names-nw-excluded-allcaps.mjs")]);
|
|
8391
|
+
const nonWesternNames = [];
|
|
8392
|
+
for (const mod of nwNameMods) for (const name of mod.default.names) nonWesternNames.push(name);
|
|
8393
|
+
if (dictionaries?.nonWesternNames) {
|
|
8394
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.nonWesternNames) : (() => {
|
|
8395
|
+
const allowed = new Set(languages.map(normalizeCorpusLanguage));
|
|
8396
|
+
return Object.entries(dictionaries.nonWesternNames).filter(([language]) => allowed.has(normalizeCorpusLanguage(language)));
|
|
8397
|
+
})();
|
|
8398
|
+
for (const [, names] of entries) for (const name of names) nonWesternNames.push(name);
|
|
8399
|
+
}
|
|
8400
|
+
const dedupNonWestern = dedup(nonWesternNames);
|
|
8401
|
+
const dedupExcludedAllCaps = dedup(nwExcludedMod.default.words);
|
|
8045
8402
|
ctx.nameCorpus = {
|
|
8046
8403
|
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
8047
8404
|
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
8048
|
-
titleTokens: Object.freeze(new Set(
|
|
8405
|
+
titleTokens: Object.freeze(new Set(dedupTitles)),
|
|
8406
|
+
titleAbbreviations: Object.freeze(titleAbbrSet),
|
|
8049
8407
|
excludedWords: Object.freeze(new Set(exclusions)),
|
|
8408
|
+
nonWesternNames: Object.freeze(new Set(dedupNonWestern)),
|
|
8409
|
+
excludedAllCaps: Object.freeze(new Set(dedupExcludedAllCaps)),
|
|
8050
8410
|
firstNamesList: Object.freeze(dedupFirst),
|
|
8051
8411
|
surnamesList: Object.freeze(dedupSurnames),
|
|
8052
|
-
titlesList: Object.freeze(
|
|
8053
|
-
excludedList: Object.freeze(exclusions)
|
|
8412
|
+
titlesList: Object.freeze(dedupTitles),
|
|
8413
|
+
excludedList: Object.freeze(exclusions),
|
|
8414
|
+
nonWesternNamesList: Object.freeze(dedupNonWestern),
|
|
8415
|
+
excludedAllCapsList: Object.freeze(dedupExcludedAllCaps)
|
|
8054
8416
|
};
|
|
8055
8417
|
} catch (err) {
|
|
8056
8418
|
ctx.nameCorpusPromise = null;
|
|
@@ -8278,6 +8640,8 @@ const TOKEN_TYPE = {
|
|
|
8278
8640
|
SURNAME: "surname",
|
|
8279
8641
|
TITLE: "title",
|
|
8280
8642
|
ABBREVIATION: "abbreviation",
|
|
8643
|
+
JA_SUFFIX: "ja_suffix",
|
|
8644
|
+
ARABIC_CONNECTOR: "arabic_connector",
|
|
8281
8645
|
CAPITALIZED: "capitalized",
|
|
8282
8646
|
OTHER: "other"
|
|
8283
8647
|
};
|
|
@@ -8304,6 +8668,141 @@ const isSurnameToken = (token, corpus) => {
|
|
|
8304
8668
|
* abbreviation: "J.", "M.", etc.
|
|
8305
8669
|
*/
|
|
8306
8670
|
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
8671
|
+
/**
|
|
8672
|
+
* Title-case a token, re-capitalizing after apostrophes
|
|
8673
|
+
* so D'Souza, O'Brien, Hon'ble match the corpus.
|
|
8674
|
+
*/
|
|
8675
|
+
const titleCaseWithApostrophe = (text) => (text[0]?.toUpperCase() ?? "") + text.slice(1).toLowerCase().replace(/'\p{Ll}/gu, (m) => m.toUpperCase());
|
|
8676
|
+
/**
|
|
8677
|
+
* Check if a token is in the non-Western name corpus.
|
|
8678
|
+
* Tries both title-cased (with apostrophe re-cap) and
|
|
8679
|
+
* raw forms.
|
|
8680
|
+
*/
|
|
8681
|
+
const isNonWesternNameToken = (token, corpus) => {
|
|
8682
|
+
if (corpus.nonWesternNames.has(token)) return true;
|
|
8683
|
+
return corpus.nonWesternNames.has(titleCaseWithApostrophe(token));
|
|
8684
|
+
};
|
|
8685
|
+
/** Japanese honorific suffixes attached via hyphen. */
|
|
8686
|
+
const JA_SUFFIXES = new Set([
|
|
8687
|
+
"san",
|
|
8688
|
+
"sama",
|
|
8689
|
+
"sensei"
|
|
8690
|
+
]);
|
|
8691
|
+
/** Arabic patronymic connectors. */
|
|
8692
|
+
const ARABIC_CONNECTORS = new Set([
|
|
8693
|
+
"bin",
|
|
8694
|
+
"bint",
|
|
8695
|
+
"ibn",
|
|
8696
|
+
"al",
|
|
8697
|
+
"el"
|
|
8698
|
+
]);
|
|
8699
|
+
/** Al-/El- prefixed name pattern (e.g., Al-Rashid, El-Amin). */
|
|
8700
|
+
const AL_EL_PREFIX_RE = /^[Aa]l-[A-Z][a-z]+$|^[Ee]l-[A-Z][a-z]+$/u;
|
|
8701
|
+
/** CJK name detection: 2-4 Han chars not adjacent to others. */
|
|
8702
|
+
const CJK_NAME_RE = /(?<!\p{Script=Han})\p{Script=Han}{2,4}(?!\p{Script=Han})/gu;
|
|
8703
|
+
/** Han character threshold for CJK-majority documents. */
|
|
8704
|
+
const CJK_HAN_RATIO = .15;
|
|
8705
|
+
const CJK_NON_PERSON_TERMS = new Set([
|
|
8706
|
+
"中国",
|
|
8707
|
+
"中國",
|
|
8708
|
+
"中文",
|
|
8709
|
+
"人民",
|
|
8710
|
+
"公司",
|
|
8711
|
+
"香港",
|
|
8712
|
+
"台湾",
|
|
8713
|
+
"臺灣",
|
|
8714
|
+
"日本",
|
|
8715
|
+
"韩国",
|
|
8716
|
+
"韓國"
|
|
8717
|
+
]);
|
|
8718
|
+
const CJK_SURNAME_CHARS = new Set([
|
|
8719
|
+
"王",
|
|
8720
|
+
"李",
|
|
8721
|
+
"张",
|
|
8722
|
+
"張",
|
|
8723
|
+
"刘",
|
|
8724
|
+
"劉",
|
|
8725
|
+
"陈",
|
|
8726
|
+
"陳",
|
|
8727
|
+
"杨",
|
|
8728
|
+
"楊",
|
|
8729
|
+
"黄",
|
|
8730
|
+
"黃",
|
|
8731
|
+
"赵",
|
|
8732
|
+
"趙",
|
|
8733
|
+
"吴",
|
|
8734
|
+
"吳",
|
|
8735
|
+
"周",
|
|
8736
|
+
"徐",
|
|
8737
|
+
"孙",
|
|
8738
|
+
"孫",
|
|
8739
|
+
"马",
|
|
8740
|
+
"馬",
|
|
8741
|
+
"朱",
|
|
8742
|
+
"胡",
|
|
8743
|
+
"郭",
|
|
8744
|
+
"何",
|
|
8745
|
+
"林",
|
|
8746
|
+
"高",
|
|
8747
|
+
"梁",
|
|
8748
|
+
"郑",
|
|
8749
|
+
"鄭",
|
|
8750
|
+
"罗",
|
|
8751
|
+
"羅",
|
|
8752
|
+
"宋",
|
|
8753
|
+
"谢",
|
|
8754
|
+
"謝",
|
|
8755
|
+
"唐",
|
|
8756
|
+
"韩",
|
|
8757
|
+
"韓",
|
|
8758
|
+
"曹",
|
|
8759
|
+
"许",
|
|
8760
|
+
"許",
|
|
8761
|
+
"邓",
|
|
8762
|
+
"鄧",
|
|
8763
|
+
"萧",
|
|
8764
|
+
"蕭",
|
|
8765
|
+
"田",
|
|
8766
|
+
"山",
|
|
8767
|
+
"佐",
|
|
8768
|
+
"鈴",
|
|
8769
|
+
"渡",
|
|
8770
|
+
"伊",
|
|
8771
|
+
"中",
|
|
8772
|
+
"小",
|
|
8773
|
+
"吉",
|
|
8774
|
+
"金",
|
|
8775
|
+
"朴",
|
|
8776
|
+
"박",
|
|
8777
|
+
"김",
|
|
8778
|
+
"이",
|
|
8779
|
+
"최",
|
|
8780
|
+
"정",
|
|
8781
|
+
"강",
|
|
8782
|
+
"조",
|
|
8783
|
+
"윤",
|
|
8784
|
+
"장",
|
|
8785
|
+
"임",
|
|
8786
|
+
"한"
|
|
8787
|
+
].join(""));
|
|
8788
|
+
const isLikelyCjkPersonName = (text) => {
|
|
8789
|
+
if (CJK_NON_PERSON_TERMS.has(text)) return false;
|
|
8790
|
+
const first = text.at(0);
|
|
8791
|
+
return first !== void 0 && CJK_SURNAME_CHARS.has(first);
|
|
8792
|
+
};
|
|
8793
|
+
/** Organization keyword filter. */
|
|
8794
|
+
const ORG_WORDS = /\b(?:Group|Company|LLC|LLP|LP|Inc|Ltd|Corp|Corporation|Holdings|Partners|Association|University|Bank|Fund|Trust|Agency|Government|Ministry|Office|Department|Council|Board|Committee|Commission|Services|Solutions|Technologies|Systems|Analytics|Software)\b/i;
|
|
8795
|
+
const isOrganization = (text) => ORG_WORDS.test(text);
|
|
8796
|
+
/** Deduplicate overlapping entity spans (first wins). */
|
|
8797
|
+
const deduplicateSpans$1 = (entities) => {
|
|
8798
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start || b.end - a.end);
|
|
8799
|
+
const result = [];
|
|
8800
|
+
for (const entity of sorted) {
|
|
8801
|
+
const last = result.at(-1);
|
|
8802
|
+
if (!last || entity.start >= last.end) result.push(entity);
|
|
8803
|
+
}
|
|
8804
|
+
return result;
|
|
8805
|
+
};
|
|
8307
8806
|
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
8308
8807
|
/**
|
|
8309
8808
|
* Split text into word segments using Intl.Segmenter.
|
|
@@ -8352,21 +8851,55 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8352
8851
|
text,
|
|
8353
8852
|
type: TOKEN_TYPE.TITLE,
|
|
8354
8853
|
start,
|
|
8854
|
+
end,
|
|
8855
|
+
...corpus.titleAbbreviations.has(stripped) ? { titleAbbreviation: true } : {}
|
|
8856
|
+
};
|
|
8857
|
+
if (JA_SUFFIXES.has(lower)) return {
|
|
8858
|
+
text,
|
|
8859
|
+
type: TOKEN_TYPE.JA_SUFFIX,
|
|
8860
|
+
start,
|
|
8355
8861
|
end
|
|
8356
8862
|
};
|
|
8863
|
+
if (ARABIC_CONNECTORS.has(lower)) return {
|
|
8864
|
+
text,
|
|
8865
|
+
type: TOKEN_TYPE.ARABIC_CONNECTOR,
|
|
8866
|
+
start,
|
|
8867
|
+
end
|
|
8868
|
+
};
|
|
8869
|
+
if (AL_EL_PREFIX_RE.test(text)) return {
|
|
8870
|
+
text,
|
|
8871
|
+
type: TOKEN_TYPE.NAME,
|
|
8872
|
+
start,
|
|
8873
|
+
end,
|
|
8874
|
+
nonWestern: true
|
|
8875
|
+
};
|
|
8357
8876
|
if (isAbbreviation(text)) return {
|
|
8358
8877
|
text,
|
|
8359
8878
|
type: TOKEN_TYPE.ABBREVIATION,
|
|
8360
8879
|
start,
|
|
8361
8880
|
end
|
|
8362
8881
|
};
|
|
8882
|
+
if (/^(?:\p{Lu}\.)+\p{Lu}?$/u.test(text)) return {
|
|
8883
|
+
text,
|
|
8884
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
8885
|
+
start,
|
|
8886
|
+
end
|
|
8887
|
+
};
|
|
8363
8888
|
if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
|
|
8364
8889
|
const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
|
|
8365
8890
|
const before = fullText.slice(lineStart, start).trimEnd();
|
|
8366
8891
|
const lastWord = /\p{L}[\p{L}\p{M}'-]*$/u.exec(before)?.[0];
|
|
8367
|
-
|
|
8368
|
-
|
|
8369
|
-
|
|
8892
|
+
const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus) || isNonWesternNameToken(token, corpus);
|
|
8893
|
+
if (lastWord && lookup(lastWord)) return {
|
|
8894
|
+
text,
|
|
8895
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
8896
|
+
start,
|
|
8897
|
+
end
|
|
8898
|
+
};
|
|
8899
|
+
const afterDot = fullText.slice(end + 1).trimStart();
|
|
8900
|
+
const nextWord = /^\p{L}[\p{L}\p{M}'-]*/u.exec(afterDot)?.[0];
|
|
8901
|
+
if (nextWord) {
|
|
8902
|
+
if (lookup(nextWord) || nextWord.length === 1 && UPPER_START_RE.test(nextWord)) return {
|
|
8370
8903
|
text,
|
|
8371
8904
|
type: TOKEN_TYPE.ABBREVIATION,
|
|
8372
8905
|
start,
|
|
@@ -8386,26 +8919,55 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8386
8919
|
start,
|
|
8387
8920
|
end
|
|
8388
8921
|
};
|
|
8389
|
-
if (text.length <
|
|
8922
|
+
if (text.length < 2) return {
|
|
8923
|
+
text,
|
|
8924
|
+
type: TOKEN_TYPE.OTHER,
|
|
8925
|
+
start,
|
|
8926
|
+
end
|
|
8927
|
+
};
|
|
8928
|
+
if (text.length < 3 && !isNonWesternNameToken(text, corpus) && !JA_SUFFIXES.has(lower) && !ARABIC_CONNECTORS.has(lower) && !(ALL_UPPER_RE.test(text) && !corpus.excludedAllCaps.has(text))) return {
|
|
8390
8929
|
text,
|
|
8391
8930
|
type: TOKEN_TYPE.OTHER,
|
|
8392
8931
|
start,
|
|
8393
8932
|
end
|
|
8394
8933
|
};
|
|
8395
8934
|
if (text.length >= 3 && ALL_UPPER_RE.test(text)) {
|
|
8935
|
+
if (corpus.excludedAllCaps.has(text)) return {
|
|
8936
|
+
text,
|
|
8937
|
+
type: TOKEN_TYPE.OTHER,
|
|
8938
|
+
start,
|
|
8939
|
+
end
|
|
8940
|
+
};
|
|
8941
|
+
const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
|
|
8942
|
+
const nwMatch = isNonWesternNameToken(titleCased, corpus);
|
|
8943
|
+
if (nwMatch && !isFirstNameToken(titleCased, corpus)) return {
|
|
8944
|
+
text,
|
|
8945
|
+
type: TOKEN_TYPE.NAME,
|
|
8946
|
+
start,
|
|
8947
|
+
end,
|
|
8948
|
+
nonWestern: true
|
|
8949
|
+
};
|
|
8396
8950
|
if (isAllCapsContextLine(fullText, start) && isAllCapsLineNameShaped(fullText, start)) {
|
|
8397
|
-
const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
|
|
8398
8951
|
if (isFirstNameToken(titleCased, corpus)) return {
|
|
8399
8952
|
text,
|
|
8400
8953
|
type: TOKEN_TYPE.NAME,
|
|
8401
8954
|
start,
|
|
8402
|
-
end
|
|
8955
|
+
end,
|
|
8956
|
+
...nwMatch ? { nonWestern: true } : {}
|
|
8403
8957
|
};
|
|
8404
8958
|
if (isSurnameToken(titleCased, corpus)) return {
|
|
8405
8959
|
text,
|
|
8406
|
-
type: TOKEN_TYPE.SURNAME,
|
|
8960
|
+
type: TOKEN_TYPE.SURNAME,
|
|
8961
|
+
start,
|
|
8962
|
+
end,
|
|
8963
|
+
...nwMatch ? { nonWestern: true } : {}
|
|
8964
|
+
};
|
|
8965
|
+
if (nwMatch) return {
|
|
8966
|
+
text,
|
|
8967
|
+
type: TOKEN_TYPE.NAME,
|
|
8407
8968
|
start,
|
|
8408
|
-
end
|
|
8969
|
+
end,
|
|
8970
|
+
nonWestern: true
|
|
8409
8971
|
};
|
|
8410
8972
|
}
|
|
8411
8973
|
return {
|
|
@@ -8421,17 +8983,32 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8421
8983
|
start,
|
|
8422
8984
|
end
|
|
8423
8985
|
};
|
|
8424
|
-
if (isFirstNameToken(text, corpus))
|
|
8986
|
+
if (isFirstNameToken(text, corpus)) {
|
|
8987
|
+
const nw = isNonWesternNameToken(text, corpus);
|
|
8988
|
+
return {
|
|
8989
|
+
text,
|
|
8990
|
+
type: TOKEN_TYPE.NAME,
|
|
8991
|
+
start,
|
|
8992
|
+
end,
|
|
8993
|
+
...nw ? { nonWestern: true } : {}
|
|
8994
|
+
};
|
|
8995
|
+
}
|
|
8996
|
+
if (isSurnameToken(text, corpus)) {
|
|
8997
|
+
const nw = isNonWesternNameToken(text, corpus);
|
|
8998
|
+
return {
|
|
8999
|
+
text,
|
|
9000
|
+
type: TOKEN_TYPE.SURNAME,
|
|
9001
|
+
start,
|
|
9002
|
+
end,
|
|
9003
|
+
...nw ? { nonWestern: true } : {}
|
|
9004
|
+
};
|
|
9005
|
+
}
|
|
9006
|
+
if (isNonWesternNameToken(text, corpus)) return {
|
|
8425
9007
|
text,
|
|
8426
9008
|
type: TOKEN_TYPE.NAME,
|
|
8427
9009
|
start,
|
|
8428
|
-
end
|
|
8429
|
-
|
|
8430
|
-
if (isSurnameToken(text, corpus)) return {
|
|
8431
|
-
text,
|
|
8432
|
-
type: TOKEN_TYPE.SURNAME,
|
|
8433
|
-
start,
|
|
8434
|
-
end
|
|
9010
|
+
end,
|
|
9011
|
+
nonWestern: true
|
|
8435
9012
|
};
|
|
8436
9013
|
return {
|
|
8437
9014
|
text,
|
|
@@ -8443,11 +9020,12 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8443
9020
|
/**
|
|
8444
9021
|
* Detect person names by looking up tokens against the
|
|
8445
9022
|
* name corpus, then chaining adjacent name-like tokens.
|
|
9023
|
+
* Handles both Western and non-Western name patterns.
|
|
8446
9024
|
*
|
|
8447
9025
|
* Requires initNameCorpus() to have been called first.
|
|
8448
9026
|
* If not initialized, returns an empty array.
|
|
8449
9027
|
*
|
|
8450
|
-
* Scoring:
|
|
9028
|
+
* Scoring (Western):
|
|
8451
9029
|
* TITLE + NAME/SURNAME → 0.95
|
|
8452
9030
|
* NAME + NAME/SURNAME → 0.9
|
|
8453
9031
|
* SURNAME + NAME/SURNAME → 0.9
|
|
@@ -8455,18 +9033,63 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8455
9033
|
* ABBREVIATION + NAME → 0.7
|
|
8456
9034
|
* Standalone NAME → 0.5 (low confidence)
|
|
8457
9035
|
* Standalone SURNAME → skip (too ambiguous)
|
|
8458
|
-
|
|
8459
|
-
|
|
9036
|
+
*
|
|
9037
|
+
* Scoring (non-Western, when chain contains nonWestern tokens):
|
|
9038
|
+
* TITLE + (nonWestern|CAPITALIZED) → 0.95
|
|
9039
|
+
* JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
|
|
9040
|
+
* ARABIC_CONNECTOR + nonWestern → 0.9
|
|
9041
|
+
* 2+ nonWestern tokens → 0.9
|
|
9042
|
+
* nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
|
|
9043
|
+
* Standalone nonWestern mid-sentence → 0.5
|
|
9044
|
+
*/
|
|
9045
|
+
const detectNameCorpus = (fullText, ctx = defaultContext, options = {}) => {
|
|
8460
9046
|
const corpus = getCorpus(ctx);
|
|
8461
9047
|
if (!corpus) return [];
|
|
8462
|
-
const
|
|
9048
|
+
const supplementalMode = options.mode === "supplemental";
|
|
8463
9049
|
const entities = [];
|
|
9050
|
+
const threshold = Math.ceil(fullText.length * CJK_HAN_RATIO);
|
|
9051
|
+
let hanCount = 0;
|
|
9052
|
+
for (const _ of fullText.matchAll(/\p{Script=Han}/gu)) {
|
|
9053
|
+
hanCount++;
|
|
9054
|
+
if (hanCount >= threshold) break;
|
|
9055
|
+
}
|
|
9056
|
+
if (fullText.length < 100 || hanCount < threshold) {
|
|
9057
|
+
CJK_NAME_RE.lastIndex = 0;
|
|
9058
|
+
let match;
|
|
9059
|
+
while ((match = CJK_NAME_RE.exec(fullText)) !== null) {
|
|
9060
|
+
const cjkText = match[0];
|
|
9061
|
+
if (isLikelyCjkPersonName(cjkText) && !isOrganization(cjkText)) entities.push({
|
|
9062
|
+
start: match.index,
|
|
9063
|
+
end: match.index + cjkText.length,
|
|
9064
|
+
label: "person",
|
|
9065
|
+
text: cjkText,
|
|
9066
|
+
score: .95,
|
|
9067
|
+
source: DETECTION_SOURCES.REGEX
|
|
9068
|
+
});
|
|
9069
|
+
}
|
|
9070
|
+
}
|
|
9071
|
+
const words = segmentWords(fullText);
|
|
9072
|
+
const tokens = [];
|
|
9073
|
+
for (let idx = 0; idx < words.length; idx++) {
|
|
9074
|
+
const word = words[idx];
|
|
9075
|
+
const lower = word.text.toLowerCase();
|
|
9076
|
+
const nextWord = words[idx + 1];
|
|
9077
|
+
if ((lower === "s" || lower === "d" || lower === "w" || lower === "r") && fullText[word.end] === "/" && nextWord && nextWord.start === word.end + 1 && nextWord.text.toLowerCase() === "o") {
|
|
9078
|
+
tokens.push({
|
|
9079
|
+
text: fullText.slice(word.start, word.end + 2),
|
|
9080
|
+
type: TOKEN_TYPE.ARABIC_CONNECTOR,
|
|
9081
|
+
start: word.start,
|
|
9082
|
+
end: word.end + 2
|
|
9083
|
+
});
|
|
9084
|
+
idx++;
|
|
9085
|
+
} else tokens.push(classifyToken(word, corpus, fullText));
|
|
9086
|
+
}
|
|
8464
9087
|
const consumed = /* @__PURE__ */ new Set();
|
|
8465
9088
|
for (let i = 0; i < tokens.length; i++) {
|
|
8466
9089
|
if (consumed.has(i)) continue;
|
|
8467
9090
|
const token = tokens[i];
|
|
8468
9091
|
if (!token) continue;
|
|
8469
|
-
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
9092
|
+
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION && token.type !== TOKEN_TYPE.ARABIC_CONNECTOR) continue;
|
|
8470
9093
|
const MAX_CHAIN = 5;
|
|
8471
9094
|
const chain = [token];
|
|
8472
9095
|
let j = i + 1;
|
|
@@ -8476,10 +9099,12 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8476
9099
|
const prev = chain.at(-1);
|
|
8477
9100
|
if (prev) {
|
|
8478
9101
|
const gap = fullText.slice(prev.end, next.start);
|
|
8479
|
-
const
|
|
9102
|
+
const periodIsPartOfPrevToken = prev.type === TOKEN_TYPE.ABBREVIATION || prev.type === TOKEN_TYPE.TITLE && prev.titleAbbreviation === true;
|
|
9103
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap) && !periodIsPartOfPrevToken;
|
|
8480
9104
|
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
9105
|
+
if (next.type === TOKEN_TYPE.JA_SUFFIX && gap !== "-" && gap.trim() !== "") break;
|
|
8481
9106
|
}
|
|
8482
|
-
if (next.type
|
|
9107
|
+
if (next.type !== TOKEN_TYPE.OTHER) {
|
|
8483
9108
|
chain.push(next);
|
|
8484
9109
|
j++;
|
|
8485
9110
|
} else break;
|
|
@@ -8488,10 +9113,24 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8488
9113
|
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
8489
9114
|
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
8490
9115
|
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
9116
|
+
const hasNonWestern = chain.some((t) => t.nonWestern === true);
|
|
9117
|
+
const hasJaSuffix = chain.some((t) => t.type === TOKEN_TYPE.JA_SUFFIX);
|
|
9118
|
+
const hasArabicConnector = chain.some((t) => t.type === TOKEN_TYPE.ARABIC_CONNECTOR);
|
|
8491
9119
|
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
8492
9120
|
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
9121
|
+
const nonWesternCount = chain.filter((t) => t.nonWestern).length;
|
|
8493
9122
|
let score;
|
|
8494
|
-
if (
|
|
9123
|
+
if (hasNonWestern) {
|
|
9124
|
+
if (hasTitle && (nonWesternCount > 0 || capitalizedCount > 0)) score = .95;
|
|
9125
|
+
else if (hasJaSuffix && (capitalizedCount > 0 || nonWesternCount > 0)) score = .9;
|
|
9126
|
+
else if (hasArabicConnector && nonWesternCount > 0) score = .9;
|
|
9127
|
+
else if (nonWesternCount >= 2) score = .9;
|
|
9128
|
+
else if (nonWesternCount > 0 && (capitalizedCount > 0 || hasAbbreviation)) score = .9;
|
|
9129
|
+
else if (nonWesternCount === 1 && chain.length === 1 && !isSentenceStart(fullText, token.start)) score = .5;
|
|
9130
|
+
else continue;
|
|
9131
|
+
if (supplementalMode && score < .9) continue;
|
|
9132
|
+
} else if (supplementalMode) continue;
|
|
9133
|
+
else if (hasTitle && hasCorpusName) score = .95;
|
|
8495
9134
|
else if (corpusCount >= 2) score = .9;
|
|
8496
9135
|
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
8497
9136
|
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
@@ -8502,7 +9141,10 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8502
9141
|
score = .5;
|
|
8503
9142
|
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
8504
9143
|
else if (hasTitle && chain.length === 1) continue;
|
|
8505
|
-
else {
|
|
9144
|
+
else if (hasJaSuffix || hasArabicConnector) {
|
|
9145
|
+
if (!hasCorpusName && !hasFirstName) continue;
|
|
9146
|
+
score = .5;
|
|
9147
|
+
} else {
|
|
8506
9148
|
if (!hasCorpusName) continue;
|
|
8507
9149
|
score = .5;
|
|
8508
9150
|
}
|
|
@@ -8512,6 +9154,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8512
9154
|
const start = first.start;
|
|
8513
9155
|
const end = last.end;
|
|
8514
9156
|
const text = fullText.slice(start, end);
|
|
9157
|
+
if (isOrganization(text)) continue;
|
|
8515
9158
|
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
8516
9159
|
entities.push({
|
|
8517
9160
|
start,
|
|
@@ -8522,7 +9165,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8522
9165
|
source: DETECTION_SOURCES.REGEX
|
|
8523
9166
|
});
|
|
8524
9167
|
}
|
|
8525
|
-
return entities;
|
|
9168
|
+
return deduplicateSpans$1(entities);
|
|
8526
9169
|
};
|
|
8527
9170
|
//#endregion
|
|
8528
9171
|
//#region src/detectors/signatures.ts
|
|
@@ -8634,219 +9277,6 @@ const detectSignatures = (fullText, _ctx = defaultContext) => {
|
|
|
8634
9277
|
return results;
|
|
8635
9278
|
};
|
|
8636
9279
|
//#endregion
|
|
8637
|
-
//#region src/config/titles.ts
|
|
8638
|
-
/**
|
|
8639
|
-
* Academic and professional title prefixes.
|
|
8640
|
-
* Plain text; the detector auto-escapes for regex.
|
|
8641
|
-
* Sorted longest-first at build time.
|
|
8642
|
-
*/
|
|
8643
|
-
const TITLE_PREFIXES = [
|
|
8644
|
-
"Ing.",
|
|
8645
|
-
"Mgr.",
|
|
8646
|
-
"MgA.",
|
|
8647
|
-
"Bc.",
|
|
8648
|
-
"BcA.",
|
|
8649
|
-
"JUDr.",
|
|
8650
|
-
"MUDr.",
|
|
8651
|
-
"MVDr.",
|
|
8652
|
-
"MDDr.",
|
|
8653
|
-
"PhDr.",
|
|
8654
|
-
"RNDr.",
|
|
8655
|
-
"PaedDr.",
|
|
8656
|
-
"ThDr.",
|
|
8657
|
-
"ThLic.",
|
|
8658
|
-
"ICDr.",
|
|
8659
|
-
"RSDr.",
|
|
8660
|
-
"PharmDr.",
|
|
8661
|
-
"artD.",
|
|
8662
|
-
"akad.",
|
|
8663
|
-
"doc.",
|
|
8664
|
-
"prof.",
|
|
8665
|
-
"ao. Univ.-Prof.",
|
|
8666
|
-
"o. Univ.-Prof.",
|
|
8667
|
-
"Univ.-Prof.",
|
|
8668
|
-
"Hon.-Prof.",
|
|
8669
|
-
"em. Prof.",
|
|
8670
|
-
"Dr. med. dent.",
|
|
8671
|
-
"Dr. med. vet.",
|
|
8672
|
-
"Dr. med.",
|
|
8673
|
-
"Dr. rer. nat.",
|
|
8674
|
-
"Dr. rer. soc.",
|
|
8675
|
-
"Dr. rer. pol.",
|
|
8676
|
-
"Dr. sc. tech.",
|
|
8677
|
-
"Dr. sc. nat.",
|
|
8678
|
-
"Dr. sc. hum.",
|
|
8679
|
-
"Dr. iur.",
|
|
8680
|
-
"Dr. jur.",
|
|
8681
|
-
"Dr. theol.",
|
|
8682
|
-
"Dr. oec.",
|
|
8683
|
-
"Dr. techn.",
|
|
8684
|
-
"Dr. h. c.",
|
|
8685
|
-
"Dr. phil.",
|
|
8686
|
-
"Dr.-Ing.",
|
|
8687
|
-
"Dr. Ing.",
|
|
8688
|
-
"Dr.",
|
|
8689
|
-
"Dipl.-Wirt.-Ing.",
|
|
8690
|
-
"Dipl.-Betriebsw.",
|
|
8691
|
-
"Dipl.-Inform.",
|
|
8692
|
-
"Dipl.-Volksw.",
|
|
8693
|
-
"Dipl.-Psych.",
|
|
8694
|
-
"Dipl.-Phys.",
|
|
8695
|
-
"Dipl.-Chem.",
|
|
8696
|
-
"Dipl.-Biol.",
|
|
8697
|
-
"Dipl.-Math.",
|
|
8698
|
-
"Dipl.-Päd.",
|
|
8699
|
-
"Dipl.-Soz.",
|
|
8700
|
-
"Dipl.-Kfm.",
|
|
8701
|
-
"Dipl.-Jur.",
|
|
8702
|
-
"Dipl. Ing.",
|
|
8703
|
-
"Dipl.-Ing.",
|
|
8704
|
-
"Mag. rer. soc. oec.",
|
|
8705
|
-
"Mag. rer. nat.",
|
|
8706
|
-
"Mag. phil.",
|
|
8707
|
-
"Mag. iur.",
|
|
8708
|
-
"Mag. arch.",
|
|
8709
|
-
"Mag. pharm.",
|
|
8710
|
-
"Mag. (FH)",
|
|
8711
|
-
"Mag.",
|
|
8712
|
-
"Bakk. rer. nat.",
|
|
8713
|
-
"Bakk. techn.",
|
|
8714
|
-
"Bakk. phil.",
|
|
8715
|
-
"Bakk.",
|
|
8716
|
-
"Lic. phil.",
|
|
8717
|
-
"Lic. iur.",
|
|
8718
|
-
"Lic. oec.",
|
|
8719
|
-
"Lic. theol.",
|
|
8720
|
-
"Lic.",
|
|
8721
|
-
"Priv.-Doz.",
|
|
8722
|
-
"PD",
|
|
8723
|
-
"RA"
|
|
8724
|
-
];
|
|
8725
|
-
/**
|
|
8726
|
-
* Courtesy/honorific titles that precede a person's
|
|
8727
|
-
* name. Sorted alphabetically. The detector escapes
|
|
8728
|
-
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
8729
|
-
*/
|
|
8730
|
-
const HONORIFICS = [
|
|
8731
|
-
"Avv.",
|
|
8732
|
-
"Dame",
|
|
8733
|
-
"Doamna",
|
|
8734
|
-
"Domnul",
|
|
8735
|
-
"Don",
|
|
8736
|
-
"Doña",
|
|
8737
|
-
"Dott.",
|
|
8738
|
-
"Judge",
|
|
8739
|
-
"Justice",
|
|
8740
|
-
"Lady",
|
|
8741
|
-
"Lord",
|
|
8742
|
-
"M.",
|
|
8743
|
-
"Madame",
|
|
8744
|
-
"Mademoiselle",
|
|
8745
|
-
"Maître",
|
|
8746
|
-
"Me",
|
|
8747
|
-
"Messrs",
|
|
8748
|
-
"Miss",
|
|
8749
|
-
"Mlle",
|
|
8750
|
-
"Mme",
|
|
8751
|
-
"Monsieur",
|
|
8752
|
-
"Mr",
|
|
8753
|
-
"Mrs",
|
|
8754
|
-
"Ms",
|
|
8755
|
-
"Pr",
|
|
8756
|
-
"Pr.",
|
|
8757
|
-
"President",
|
|
8758
|
-
"Señor",
|
|
8759
|
-
"Señora",
|
|
8760
|
-
"Sig.",
|
|
8761
|
-
"Sig.ra",
|
|
8762
|
-
"Signor",
|
|
8763
|
-
"Signora",
|
|
8764
|
-
"Signorina",
|
|
8765
|
-
"Sir",
|
|
8766
|
-
"Sr.",
|
|
8767
|
-
"Sra."
|
|
8768
|
-
];
|
|
8769
|
-
/**
|
|
8770
|
-
* Honorifics that need \b word-boundary anchors
|
|
8771
|
-
* (short or common words that could match mid-word).
|
|
8772
|
-
*/
|
|
8773
|
-
const HONORIFIC_BOUNDARY = new Set([
|
|
8774
|
-
"Don",
|
|
8775
|
-
"Doña",
|
|
8776
|
-
"M.",
|
|
8777
|
-
"Me",
|
|
8778
|
-
"Pr",
|
|
8779
|
-
"Pr.",
|
|
8780
|
-
"Señor",
|
|
8781
|
-
"Señora"
|
|
8782
|
-
]);
|
|
8783
|
-
/**
|
|
8784
|
-
* Honorifics that are abbreviations: a dot after them is an
|
|
8785
|
-
* abbreviation dot (same sentence), so the detector keeps an
|
|
8786
|
-
* optional `.` between the title and the name ("Mr. Smith").
|
|
8787
|
-
* Every other honorific is a full word — a trailing dot ends a
|
|
8788
|
-
* sentence, so the detector must NOT consume it (otherwise a span
|
|
8789
|
-
* like "President. The Employee" crosses the sentence boundary).
|
|
8790
|
-
*
|
|
8791
|
-
* Maintained explicitly, NOT derived from "ends in a dot": "Mr" is
|
|
8792
|
-
* an abbreviation written without a dot, and "Lord" is a full word.
|
|
8793
|
-
*/
|
|
8794
|
-
const HONORIFIC_ABBREVIATION = new Set([
|
|
8795
|
-
"Avv.",
|
|
8796
|
-
"Dott.",
|
|
8797
|
-
"M.",
|
|
8798
|
-
"Me",
|
|
8799
|
-
"Messrs",
|
|
8800
|
-
"Mlle",
|
|
8801
|
-
"Mme",
|
|
8802
|
-
"Mr",
|
|
8803
|
-
"Mrs",
|
|
8804
|
-
"Ms",
|
|
8805
|
-
"Pr",
|
|
8806
|
-
"Pr.",
|
|
8807
|
-
"Sig.",
|
|
8808
|
-
"Sig.ra",
|
|
8809
|
-
"Sr.",
|
|
8810
|
-
"Sra."
|
|
8811
|
-
]);
|
|
8812
|
-
/**
|
|
8813
|
-
* Post-nominal degrees (comma or space separated after name).
|
|
8814
|
-
* Plain text; the detector auto-escapes for regex.
|
|
8815
|
-
*/
|
|
8816
|
-
const POST_NOMINALS = [
|
|
8817
|
-
"Ph.D.",
|
|
8818
|
-
"Ph.D",
|
|
8819
|
-
"CSc.",
|
|
8820
|
-
"DrSc.",
|
|
8821
|
-
"ArtD.",
|
|
8822
|
-
"D.Phil.",
|
|
8823
|
-
"DPhil.",
|
|
8824
|
-
"MPhil.",
|
|
8825
|
-
"MBA",
|
|
8826
|
-
"MPA",
|
|
8827
|
-
"LL.M.",
|
|
8828
|
-
"LL.B.",
|
|
8829
|
-
"M.Sc.",
|
|
8830
|
-
"B.Sc.",
|
|
8831
|
-
"MSc.",
|
|
8832
|
-
"BSc.",
|
|
8833
|
-
"M.Eng.",
|
|
8834
|
-
"B.Eng.",
|
|
8835
|
-
"M.A.",
|
|
8836
|
-
"B.A.",
|
|
8837
|
-
"JCD",
|
|
8838
|
-
"JD",
|
|
8839
|
-
"DiS.",
|
|
8840
|
-
"ACCA",
|
|
8841
|
-
"FCCA",
|
|
8842
|
-
"CIPM",
|
|
8843
|
-
"CIPT",
|
|
8844
|
-
"CIPP/E",
|
|
8845
|
-
"CIPP",
|
|
8846
|
-
"KC",
|
|
8847
|
-
"QC"
|
|
8848
|
-
];
|
|
8849
|
-
//#endregion
|
|
8850
9280
|
//#region src/data/amount-words.json
|
|
8851
9281
|
var amount_words_exports = /* @__PURE__ */ __exportAll({
|
|
8852
9282
|
_comment: () => _comment$1,
|
|
@@ -10721,6 +11151,24 @@ const TRIGGER_LOOKAHEAD_MARGIN = 128;
|
|
|
10721
11151
|
const LINE_TRIGGER_LOOKAHEAD = 2048;
|
|
10722
11152
|
const MATCH_PATTERN_LOOKAHEAD = 512;
|
|
10723
11153
|
const PHONE_VALUE_START_RE = /^[+(\d]/;
|
|
11154
|
+
/**
|
|
11155
|
+
* Phone-shaped run: an opening "+", "(", or digit, then
|
|
11156
|
+
* any mix of digits, whitespace, parens, dots, slashes,
|
|
11157
|
+
* and dash variants. The dash class includes every
|
|
11158
|
+
* typographic dash (U+2011 non-breaking hyphen, U+2013
|
|
11159
|
+
* en-dash, etc.) via `DASH_INNER` so a Unicode separator
|
|
11160
|
+
* ("+1 555‑123‑4567") does not break the run and leave the
|
|
11161
|
+
* number unredacted.
|
|
11162
|
+
*/
|
|
11163
|
+
const PHONE_SHAPE_PREFIX_RE = new RegExp(`^[+(\\d][\\d\\s()./${DASH_INNER}]*`);
|
|
11164
|
+
/**
|
|
11165
|
+
* Optional phone-extension suffix following the numeric
|
|
11166
|
+
* run: "ext", "ext.", "extension", or "x" (case-
|
|
11167
|
+
* insensitive) plus 1-6 digits, with optional leading
|
|
11168
|
+
* whitespace. Matches "+1 555 123 4567 ext. 89" and
|
|
11169
|
+
* "555-1234 x42" so the shape bound keeps the extension.
|
|
11170
|
+
*/
|
|
11171
|
+
const PHONE_EXTENSION_SUFFIX_RE = /^\s*(?:ext\.?|extension|x)\s*\d{1,6}/i;
|
|
10724
11172
|
const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
|
|
10725
11173
|
const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
10726
11174
|
const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
@@ -10813,6 +11261,19 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
10813
11261
|
end = inlineLabel.index;
|
|
10814
11262
|
foundLineStop = true;
|
|
10815
11263
|
}
|
|
11264
|
+
const region = valueText.slice(0, end);
|
|
11265
|
+
const sentenceBreak = /\.\s/.exec(region);
|
|
11266
|
+
const shapeRegion = sentenceBreak !== null ? region.slice(0, sentenceBreak.index) : region;
|
|
11267
|
+
const shape = PHONE_SHAPE_PREFIX_RE.exec(shapeRegion);
|
|
11268
|
+
if (shape) {
|
|
11269
|
+
let shapeEnd = shape[0].replace(/\D+$/, "").length;
|
|
11270
|
+
const extension = PHONE_EXTENSION_SUFFIX_RE.exec(region.slice(shapeEnd));
|
|
11271
|
+
if (extension !== null) shapeEnd += extension[0].length;
|
|
11272
|
+
if (shapeEnd > 0 && shapeEnd < end) {
|
|
11273
|
+
end = shapeEnd > MAX_TRIGGER_VALUE_LEN ? Math.min(capAtWordBoundary(valueText, MAX_TRIGGER_VALUE_LEN), MAX_TRIGGER_VALUE_LEN) : shapeEnd;
|
|
11274
|
+
foundLineStop = true;
|
|
11275
|
+
}
|
|
11276
|
+
}
|
|
10816
11277
|
}
|
|
10817
11278
|
if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
|
|
10818
11279
|
const rawSlice = valueText.slice(0, end);
|
|
@@ -10991,10 +11452,20 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
10991
11452
|
if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
|
|
10992
11453
|
const triggerEnd = match.end;
|
|
10993
11454
|
const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
|
|
10994
|
-
|
|
11455
|
+
let value = rawValue ? stripQuotes(rawValue) : null;
|
|
10995
11456
|
if (value) {
|
|
10996
11457
|
if (!applyValidations(value.text, rule.validations)) continue;
|
|
10997
11458
|
if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
|
|
11459
|
+
if (rule.label === "phone number" && value.text.length > MAX_TRIGGER_VALUE_LEN && fullText[value.end] !== "\n" && fullText[value.end] !== " ") {
|
|
11460
|
+
const cappedEnd = Math.min(capAtWordBoundary(value.text, MAX_TRIGGER_VALUE_LEN), MAX_TRIGGER_VALUE_LEN);
|
|
11461
|
+
const cappedText = value.text.slice(0, cappedEnd).trimEnd();
|
|
11462
|
+
if (cappedText.length === 0) continue;
|
|
11463
|
+
value = {
|
|
11464
|
+
...value,
|
|
11465
|
+
end: value.start + cappedText.length,
|
|
11466
|
+
text: cappedText
|
|
11467
|
+
};
|
|
11468
|
+
}
|
|
10998
11469
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
10999
11470
|
let entityEnd = value.end;
|
|
11000
11471
|
let entityText = fullText.slice(entityStart, entityEnd);
|
|
@@ -11892,6 +12363,7 @@ const patternSources = (sources) => {
|
|
|
11892
12363
|
if (sources === void 0) return EMPTY_PATTERN_SOURCES;
|
|
11893
12364
|
return Array.isArray(sources) ? sources : [sources];
|
|
11894
12365
|
};
|
|
12366
|
+
const hasPersonNameSource = (match) => match.sources.includes("first-name") || match.sources.includes("surname");
|
|
11895
12367
|
const addPatternLabel = (list, index, label) => {
|
|
11896
12368
|
const existing = list[index];
|
|
11897
12369
|
if (existing === void 0) {
|
|
@@ -12238,6 +12710,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
12238
12710
|
const first = chain.at(0);
|
|
12239
12711
|
const last = chain.at(-1);
|
|
12240
12712
|
if (!first || !last) continue;
|
|
12713
|
+
if (!chain.some(hasPersonNameSource)) continue;
|
|
12241
12714
|
if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
|
|
12242
12715
|
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
12243
12716
|
const score = chain.length >= 2 ? .9 : .5;
|
|
@@ -13820,6 +14293,7 @@ const applyHotwordRules = (entities, fullText) => {
|
|
|
13820
14293
|
/** Max gap (in chars) between entities to merge. */
|
|
13821
14294
|
const MAX_GAP = 3;
|
|
13822
14295
|
const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
14296
|
+
const hasDetectorLockedBoundary = (entity) => entity.label === "phone number" && entity.source === DETECTION_SOURCES.TRIGGER;
|
|
13823
14297
|
/**
|
|
13824
14298
|
* Characters allowed in the gap between two adjacent
|
|
13825
14299
|
* same-label entities that should be merged: spaces,
|
|
@@ -14000,7 +14474,7 @@ const fixPartialWords = (entities, fullText) => {
|
|
|
14000
14474
|
})).sort((a, b) => a.entity.end - b.entity.end);
|
|
14001
14475
|
const endPositions = byEnd.map((x) => x.entity.end);
|
|
14002
14476
|
return sorted.map((e, eIdx) => {
|
|
14003
|
-
if (hasLockedBoundary$1(e)) return e;
|
|
14477
|
+
if (hasLockedBoundary$1(e) || hasDetectorLockedBoundary(e)) return e;
|
|
14004
14478
|
if (e.text !== fullText.slice(e.start, e.end)) return e;
|
|
14005
14479
|
let newStart = wordStartAt(e.start, boundaries, fullText);
|
|
14006
14480
|
let newEnd = wordEndAt(e.end, boundaries, fullText);
|
|
@@ -14403,6 +14877,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
14403
14877
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
14404
14878
|
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
14405
14879
|
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
14880
|
+
const isCoveredByDenyListEntity = (entity, denyListEntity) => entity.label === denyListEntity.label && denyListEntity.start <= entity.start && denyListEntity.end >= entity.end;
|
|
14881
|
+
const filterNameCorpusCoveredByDenyList = (nameCorpusEntities, denyListEntities) => nameCorpusEntities.filter((entity) => !denyListEntities.some((denyListEntity) => isCoveredByDenyListEntity(entity, denyListEntity)));
|
|
14406
14882
|
const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
|
|
14407
14883
|
const BARE_POSTAL_CODE_RE = /^\s*(?:\d{3}\s?\d{2}|\d{2}[-–]\d{3}|\d{5}(?:[-–]\d{3,4})?)\s*$/u;
|
|
14408
14884
|
const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
|
|
@@ -14923,6 +15399,13 @@ const runPipeline = async (options) => {
|
|
|
14923
15399
|
const rawDenyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
|
|
14924
15400
|
const denyListEntities = filterAllowedLabels(rawDenyListEntities, preHotwordAllowedLabels);
|
|
14925
15401
|
if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
|
|
15402
|
+
if (config.enableNameCorpus && config.enableDenyList) {
|
|
15403
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
15404
|
+
checkAbort(signal);
|
|
15405
|
+
rawNameCorpusEntities = filterNameCorpusCoveredByDenyList(detectNameCorpus(fullText, ctx, { mode: "supplemental" }), rawDenyListEntities);
|
|
15406
|
+
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|
|
15407
|
+
log("name-corpus", `${nameCorpusEntities.length} supplemental matches`);
|
|
15408
|
+
}
|
|
14926
15409
|
const rawGazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
|
|
14927
15410
|
const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
|
|
14928
15411
|
if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
|
|
@@ -15618,6 +16101,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
15618
16101
|
//#region src/wasm.ts
|
|
15619
16102
|
initTextSearch(TextSearch);
|
|
15620
16103
|
//#endregion
|
|
15621
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
16104
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
15622
16105
|
|
|
15623
16106
|
//# sourceMappingURL=wasm.mjs.map
|