@stll/anonymize-wasm 1.4.8 → 1.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/names-nw-excluded-allcaps.mjs +112 -0
- package/dist/names-nw-excluded-allcaps.mjs.map +1 -0
- package/dist/wasm.d.mts +85 -28
- package/dist/wasm.mjs +1018 -323
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -33,6 +33,10 @@ const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
|
33
33
|
* shallow copies (spread). Uses position + label so the
|
|
34
34
|
* key is identical for the original object and any
|
|
35
35
|
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
36
|
+
*
|
|
37
|
+
* @deprecated No longer used internally: coref alias
|
|
38
|
+
* links travel on the entities themselves
|
|
39
|
+
* (`corefSourceText`). Kept for API compatibility.
|
|
36
40
|
*/
|
|
37
41
|
const corefKey = (e) => `${e.start}:${e.end}:${e.label}`;
|
|
38
42
|
/** Create a fresh, empty pipeline context. */
|
|
@@ -62,8 +66,7 @@ const createPipelineContext = () => ({
|
|
|
62
66
|
roleStopSetPromise: null,
|
|
63
67
|
zoneHeadingPatterns: null,
|
|
64
68
|
zoneSigningPatterns: null,
|
|
65
|
-
zoneInitPromise: null
|
|
66
|
-
corefSourceMap: /* @__PURE__ */ new Map()
|
|
69
|
+
zoneInitPromise: null
|
|
67
70
|
});
|
|
68
71
|
/**
|
|
69
72
|
* Module-level default context. Used when callers
|
|
@@ -1761,11 +1764,14 @@ const isWordChar = (ch) => {
|
|
|
1761
1764
|
* character before the start and after the end are NOT
|
|
1762
1765
|
* word characters (letter/digit).
|
|
1763
1766
|
*
|
|
1764
|
-
*
|
|
1765
|
-
*
|
|
1766
|
-
*
|
|
1767
|
+
* Each returned alias carries `corefSourceText` linking
|
|
1768
|
+
* it to its source entity text, for consistent
|
|
1769
|
+
* placeholder numbering.
|
|
1770
|
+
*
|
|
1771
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
1772
|
+
* alias links now travel on the entities themselves.
|
|
1767
1773
|
*/
|
|
1768
|
-
const findCoreferenceSpans = (fullText, terms,
|
|
1774
|
+
const findCoreferenceSpans = (fullText, terms, _ctx = defaultContext) => {
|
|
1769
1775
|
const results = [];
|
|
1770
1776
|
for (const term of terms) {
|
|
1771
1777
|
let searchFrom = 0;
|
|
@@ -1779,16 +1785,15 @@ const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
|
1779
1785
|
searchFrom = idx + 1;
|
|
1780
1786
|
continue;
|
|
1781
1787
|
}
|
|
1782
|
-
|
|
1788
|
+
results.push({
|
|
1783
1789
|
start: idx,
|
|
1784
1790
|
end: matchEnd,
|
|
1785
1791
|
label: term.label,
|
|
1786
1792
|
text: term.alias,
|
|
1787
1793
|
score: .95,
|
|
1788
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
results.push(entity);
|
|
1794
|
+
source: DETECTION_SOURCES.COREFERENCE,
|
|
1795
|
+
corefSourceText: term.sourceText
|
|
1796
|
+
});
|
|
1792
1797
|
searchFrom = matchEnd;
|
|
1793
1798
|
}
|
|
1794
1799
|
}
|
|
@@ -7954,6 +7959,308 @@ const processCountryMatches = (allMatches, sliceStart, sliceEnd, fullText, data)
|
|
|
7954
7959
|
return results;
|
|
7955
7960
|
};
|
|
7956
7961
|
//#endregion
|
|
7962
|
+
//#region src/config/titles.ts
|
|
7963
|
+
/**
|
|
7964
|
+
* Academic and professional title prefixes.
|
|
7965
|
+
* Plain text; the detector auto-escapes for regex.
|
|
7966
|
+
* Sorted longest-first at build time.
|
|
7967
|
+
*/
|
|
7968
|
+
const TITLE_PREFIXES = [
|
|
7969
|
+
"Ing.",
|
|
7970
|
+
"Mgr.",
|
|
7971
|
+
"MgA.",
|
|
7972
|
+
"Bc.",
|
|
7973
|
+
"BcA.",
|
|
7974
|
+
"JUDr.",
|
|
7975
|
+
"MUDr.",
|
|
7976
|
+
"MVDr.",
|
|
7977
|
+
"MDDr.",
|
|
7978
|
+
"PhDr.",
|
|
7979
|
+
"RNDr.",
|
|
7980
|
+
"PaedDr.",
|
|
7981
|
+
"ThDr.",
|
|
7982
|
+
"ThLic.",
|
|
7983
|
+
"ICDr.",
|
|
7984
|
+
"RSDr.",
|
|
7985
|
+
"PharmDr.",
|
|
7986
|
+
"artD.",
|
|
7987
|
+
"akad.",
|
|
7988
|
+
"doc.",
|
|
7989
|
+
"prof.",
|
|
7990
|
+
"ao. Univ.-Prof.",
|
|
7991
|
+
"o. Univ.-Prof.",
|
|
7992
|
+
"Univ.-Prof.",
|
|
7993
|
+
"Hon.-Prof.",
|
|
7994
|
+
"em. Prof.",
|
|
7995
|
+
"Dr. med. dent.",
|
|
7996
|
+
"Dr. med. vet.",
|
|
7997
|
+
"Dr. med.",
|
|
7998
|
+
"Dr. rer. nat.",
|
|
7999
|
+
"Dr. rer. soc.",
|
|
8000
|
+
"Dr. rer. pol.",
|
|
8001
|
+
"Dr. sc. tech.",
|
|
8002
|
+
"Dr. sc. nat.",
|
|
8003
|
+
"Dr. sc. hum.",
|
|
8004
|
+
"Dr. iur.",
|
|
8005
|
+
"Dr. jur.",
|
|
8006
|
+
"Dr. theol.",
|
|
8007
|
+
"Dr. oec.",
|
|
8008
|
+
"Dr. techn.",
|
|
8009
|
+
"Dr. h. c.",
|
|
8010
|
+
"Dr. phil.",
|
|
8011
|
+
"Dr.-Ing.",
|
|
8012
|
+
"Dr. Ing.",
|
|
8013
|
+
"Dr.",
|
|
8014
|
+
"Dipl.-Wirt.-Ing.",
|
|
8015
|
+
"Dipl.-Betriebsw.",
|
|
8016
|
+
"Dipl.-Inform.",
|
|
8017
|
+
"Dipl.-Volksw.",
|
|
8018
|
+
"Dipl.-Psych.",
|
|
8019
|
+
"Dipl.-Phys.",
|
|
8020
|
+
"Dipl.-Chem.",
|
|
8021
|
+
"Dipl.-Biol.",
|
|
8022
|
+
"Dipl.-Math.",
|
|
8023
|
+
"Dipl.-Päd.",
|
|
8024
|
+
"Dipl.-Soz.",
|
|
8025
|
+
"Dipl.-Kfm.",
|
|
8026
|
+
"Dipl.-Jur.",
|
|
8027
|
+
"Dipl. Ing.",
|
|
8028
|
+
"Dipl.-Ing.",
|
|
8029
|
+
"Mag. rer. soc. oec.",
|
|
8030
|
+
"Mag. rer. nat.",
|
|
8031
|
+
"Mag. phil.",
|
|
8032
|
+
"Mag. iur.",
|
|
8033
|
+
"Mag. arch.",
|
|
8034
|
+
"Mag. pharm.",
|
|
8035
|
+
"Mag. (FH)",
|
|
8036
|
+
"Mag.",
|
|
8037
|
+
"Bakk. rer. nat.",
|
|
8038
|
+
"Bakk. techn.",
|
|
8039
|
+
"Bakk. phil.",
|
|
8040
|
+
"Bakk.",
|
|
8041
|
+
"Lic. phil.",
|
|
8042
|
+
"Lic. iur.",
|
|
8043
|
+
"Lic. oec.",
|
|
8044
|
+
"Lic. theol.",
|
|
8045
|
+
"Lic.",
|
|
8046
|
+
"Priv.-Doz.",
|
|
8047
|
+
"PD",
|
|
8048
|
+
"RA"
|
|
8049
|
+
];
|
|
8050
|
+
/**
|
|
8051
|
+
* Courtesy/honorific titles that precede a person's
|
|
8052
|
+
* name. Sorted alphabetically. The detector escapes
|
|
8053
|
+
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
8054
|
+
*/
|
|
8055
|
+
const HONORIFICS = [
|
|
8056
|
+
"Avv.",
|
|
8057
|
+
"Dame",
|
|
8058
|
+
"Doamna",
|
|
8059
|
+
"Domnul",
|
|
8060
|
+
"Don",
|
|
8061
|
+
"Doña",
|
|
8062
|
+
"Dott.",
|
|
8063
|
+
"Judge",
|
|
8064
|
+
"Justice",
|
|
8065
|
+
"Lady",
|
|
8066
|
+
"Lord",
|
|
8067
|
+
"M.",
|
|
8068
|
+
"Madame",
|
|
8069
|
+
"Mademoiselle",
|
|
8070
|
+
"Maître",
|
|
8071
|
+
"Me",
|
|
8072
|
+
"Messrs",
|
|
8073
|
+
"Miss",
|
|
8074
|
+
"Mlle",
|
|
8075
|
+
"Mme",
|
|
8076
|
+
"Monsieur",
|
|
8077
|
+
"Mr",
|
|
8078
|
+
"Mrs",
|
|
8079
|
+
"Ms",
|
|
8080
|
+
"Pr",
|
|
8081
|
+
"Pr.",
|
|
8082
|
+
"President",
|
|
8083
|
+
"Señor",
|
|
8084
|
+
"Señora",
|
|
8085
|
+
"Sig.",
|
|
8086
|
+
"Sig.ra",
|
|
8087
|
+
"Signor",
|
|
8088
|
+
"Signora",
|
|
8089
|
+
"Signorina",
|
|
8090
|
+
"Sir",
|
|
8091
|
+
"Sr.",
|
|
8092
|
+
"Sra."
|
|
8093
|
+
];
|
|
8094
|
+
/**
|
|
8095
|
+
* Honorifics that need \b word-boundary anchors
|
|
8096
|
+
* (short or common words that could match mid-word).
|
|
8097
|
+
*/
|
|
8098
|
+
const HONORIFIC_BOUNDARY = new Set([
|
|
8099
|
+
"Don",
|
|
8100
|
+
"Doña",
|
|
8101
|
+
"M.",
|
|
8102
|
+
"Me",
|
|
8103
|
+
"Pr",
|
|
8104
|
+
"Pr.",
|
|
8105
|
+
"Señor",
|
|
8106
|
+
"Señora"
|
|
8107
|
+
]);
|
|
8108
|
+
/**
|
|
8109
|
+
* Honorifics that are abbreviations: a dot after them is an
|
|
8110
|
+
* abbreviation dot (same sentence), so the detector keeps an
|
|
8111
|
+
* optional `.` between the title and the name ("Mr. Smith").
|
|
8112
|
+
* Every other honorific is a full word; a trailing dot ends a
|
|
8113
|
+
* sentence, so the detector must NOT consume it (otherwise a span
|
|
8114
|
+
* like "President. The Employee" crosses the sentence boundary).
|
|
8115
|
+
*
|
|
8116
|
+
* Maintained explicitly, NOT derived from "ends in a dot": "Mr" is
|
|
8117
|
+
* an abbreviation written without a dot, and "Lord" is a full word.
|
|
8118
|
+
*/
|
|
8119
|
+
const HONORIFIC_ABBREVIATION = new Set([
|
|
8120
|
+
"Avv.",
|
|
8121
|
+
"Dott.",
|
|
8122
|
+
"M.",
|
|
8123
|
+
"Me",
|
|
8124
|
+
"Messrs",
|
|
8125
|
+
"Mlle",
|
|
8126
|
+
"Mme",
|
|
8127
|
+
"Mr",
|
|
8128
|
+
"Mrs",
|
|
8129
|
+
"Ms",
|
|
8130
|
+
"Pr",
|
|
8131
|
+
"Pr.",
|
|
8132
|
+
"Sig.",
|
|
8133
|
+
"Sig.ra",
|
|
8134
|
+
"Sr.",
|
|
8135
|
+
"Sra."
|
|
8136
|
+
]);
|
|
8137
|
+
/**
|
|
8138
|
+
* Post-nominal degrees (comma or space separated after name).
|
|
8139
|
+
* Plain text; the detector auto-escapes for regex.
|
|
8140
|
+
*/
|
|
8141
|
+
const POST_NOMINALS = [
|
|
8142
|
+
"Ph.D.",
|
|
8143
|
+
"Ph.D",
|
|
8144
|
+
"CSc.",
|
|
8145
|
+
"DrSc.",
|
|
8146
|
+
"ArtD.",
|
|
8147
|
+
"D.Phil.",
|
|
8148
|
+
"DPhil.",
|
|
8149
|
+
"MPhil.",
|
|
8150
|
+
"MBA",
|
|
8151
|
+
"MPA",
|
|
8152
|
+
"LL.M.",
|
|
8153
|
+
"LL.B.",
|
|
8154
|
+
"M.Sc.",
|
|
8155
|
+
"B.Sc.",
|
|
8156
|
+
"MSc.",
|
|
8157
|
+
"BSc.",
|
|
8158
|
+
"M.Eng.",
|
|
8159
|
+
"B.Eng.",
|
|
8160
|
+
"M.A.",
|
|
8161
|
+
"B.A.",
|
|
8162
|
+
"JCD",
|
|
8163
|
+
"JD",
|
|
8164
|
+
"DiS.",
|
|
8165
|
+
"ACCA",
|
|
8166
|
+
"FCCA",
|
|
8167
|
+
"CIPM",
|
|
8168
|
+
"CIPT",
|
|
8169
|
+
"CIPP/E",
|
|
8170
|
+
"CIPP",
|
|
8171
|
+
"KC",
|
|
8172
|
+
"QC"
|
|
8173
|
+
];
|
|
8174
|
+
const NONWESTERN_HONORIFICS = {
|
|
8175
|
+
in: [
|
|
8176
|
+
"Sri",
|
|
8177
|
+
"Shri",
|
|
8178
|
+
"Smt",
|
|
8179
|
+
"Smt.",
|
|
8180
|
+
"Kumari",
|
|
8181
|
+
"Pandit",
|
|
8182
|
+
"Pt.",
|
|
8183
|
+
"Adv.",
|
|
8184
|
+
"Adv",
|
|
8185
|
+
"Justice",
|
|
8186
|
+
"Hon'ble"
|
|
8187
|
+
],
|
|
8188
|
+
ar: [
|
|
8189
|
+
"Al-",
|
|
8190
|
+
"El-",
|
|
8191
|
+
"Sheikh",
|
|
8192
|
+
"Shaikh",
|
|
8193
|
+
"Sheikha",
|
|
8194
|
+
"Ustaz",
|
|
8195
|
+
"Ustaza",
|
|
8196
|
+
"Abu",
|
|
8197
|
+
"Umm"
|
|
8198
|
+
],
|
|
8199
|
+
"zh-Latn": [
|
|
8200
|
+
"Encik",
|
|
8201
|
+
"Puan",
|
|
8202
|
+
"Datuk",
|
|
8203
|
+
"Dato'",
|
|
8204
|
+
"Dato",
|
|
8205
|
+
"Tan Sri",
|
|
8206
|
+
"Tun"
|
|
8207
|
+
],
|
|
8208
|
+
"ja-Latn": [
|
|
8209
|
+
"-san",
|
|
8210
|
+
"-sama",
|
|
8211
|
+
"-sensei",
|
|
8212
|
+
"Mr.",
|
|
8213
|
+
"Ms."
|
|
8214
|
+
],
|
|
8215
|
+
ko: ["Sunsaeng", "Gyosu"],
|
|
8216
|
+
th: [
|
|
8217
|
+
"Khun",
|
|
8218
|
+
"Nai",
|
|
8219
|
+
"Nang",
|
|
8220
|
+
"Khunying",
|
|
8221
|
+
"Luang",
|
|
8222
|
+
"Phra",
|
|
8223
|
+
"Mom",
|
|
8224
|
+
"Momluang",
|
|
8225
|
+
"Mommuen",
|
|
8226
|
+
"Momratchawong"
|
|
8227
|
+
],
|
|
8228
|
+
vi: [
|
|
8229
|
+
"Ông",
|
|
8230
|
+
"Ong",
|
|
8231
|
+
"Ba",
|
|
8232
|
+
"Co"
|
|
8233
|
+
],
|
|
8234
|
+
fil: [
|
|
8235
|
+
"Atty.",
|
|
8236
|
+
"Atty",
|
|
8237
|
+
"Ginoo",
|
|
8238
|
+
"Ginang",
|
|
8239
|
+
"Binibini",
|
|
8240
|
+
"G.",
|
|
8241
|
+
"Gng.",
|
|
8242
|
+
"Bb."
|
|
8243
|
+
],
|
|
8244
|
+
id: [
|
|
8245
|
+
"Bapak",
|
|
8246
|
+
"Ibu",
|
|
8247
|
+
"Pak",
|
|
8248
|
+
"Bu",
|
|
8249
|
+
"Raden",
|
|
8250
|
+
"Tuan",
|
|
8251
|
+
"Nyonya",
|
|
8252
|
+
"Nona",
|
|
8253
|
+
"Haji",
|
|
8254
|
+
"Hajjah",
|
|
8255
|
+
"H.",
|
|
8256
|
+
"Hj.",
|
|
8257
|
+
"S.H.",
|
|
8258
|
+
"S.H",
|
|
8259
|
+
"Ir.",
|
|
8260
|
+
"Drs."
|
|
8261
|
+
]
|
|
8262
|
+
};
|
|
8263
|
+
//#endregion
|
|
7957
8264
|
//#region src/util/text.ts
|
|
7958
8265
|
/**
|
|
7959
8266
|
* Shared text utilities for detectors.
|
|
@@ -7985,6 +8292,30 @@ const getCorpus = (ctx) => ctx.nameCorpus;
|
|
|
7985
8292
|
const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstNamesList ?? [];
|
|
7986
8293
|
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
7987
8294
|
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
8295
|
+
const getNameCorpusNonWesternNames = (ctx = defaultContext) => ctx.nameCorpus?.nonWesternNamesList ?? [];
|
|
8296
|
+
const NONWESTERN_LOCALE_KEYS = [
|
|
8297
|
+
"in",
|
|
8298
|
+
"ar",
|
|
8299
|
+
"ja-latn",
|
|
8300
|
+
"ko",
|
|
8301
|
+
"zh-latn",
|
|
8302
|
+
"th",
|
|
8303
|
+
"vi",
|
|
8304
|
+
"fil",
|
|
8305
|
+
"id"
|
|
8306
|
+
];
|
|
8307
|
+
const normalizeCorpusLanguage = (language) => language.toLowerCase();
|
|
8308
|
+
const getScopedNonWesternLocaleKeys = (languages) => {
|
|
8309
|
+
if (languages === void 0) return NONWESTERN_LOCALE_KEYS;
|
|
8310
|
+
const allowed = new Set(languages.map(normalizeCorpusLanguage));
|
|
8311
|
+
return NONWESTERN_LOCALE_KEYS.filter((locale) => allowed.has(locale));
|
|
8312
|
+
};
|
|
8313
|
+
const getScopedNonWesternHonorifics = (languages) => {
|
|
8314
|
+
const entries = Object.entries(NONWESTERN_HONORIFICS);
|
|
8315
|
+
if (languages === void 0) return entries.flatMap(([, forms]) => forms);
|
|
8316
|
+
const allowed = new Set(languages.map(normalizeCorpusLanguage));
|
|
8317
|
+
return entries.filter(([locale]) => allowed.has(normalizeCorpusLanguage(locale))).flatMap(([, forms]) => forms);
|
|
8318
|
+
};
|
|
7988
8319
|
/**
|
|
7989
8320
|
* Load name corpus data from injected dictionaries
|
|
7990
8321
|
* and legacy config files. Merges all sources.
|
|
@@ -8035,17 +8366,42 @@ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
|
8035
8366
|
const commonWords = new Set(commonWordsMod.default.words.map((word) => word.toLowerCase()));
|
|
8036
8367
|
const dedupFirst = dedup(firstNames);
|
|
8037
8368
|
const dedupSurnames = dedup(surnames).filter((name) => !commonWords.has(name.toLowerCase()));
|
|
8038
|
-
const titles = titleMod.default.tokens;
|
|
8369
|
+
const titles = [...titleMod.default.tokens];
|
|
8370
|
+
const scopedNonWesternHonorifics = getScopedNonWesternHonorifics(languages);
|
|
8371
|
+
for (const form of scopedNonWesternHonorifics) titles.push(form.replace(/[.-]+$/u, "").toLowerCase());
|
|
8372
|
+
const dedupTitles = dedup(titles);
|
|
8373
|
+
const titleAbbrSet = /* @__PURE__ */ new Set();
|
|
8374
|
+
for (const prefix of TITLE_PREFIXES) if (prefix.endsWith(".")) titleAbbrSet.add(prefix.replace(/[.-]+$/u, "").toLowerCase());
|
|
8375
|
+
for (const form of HONORIFIC_ABBREVIATION) titleAbbrSet.add(form.replace(/[.-]+$/u, "").toLowerCase());
|
|
8376
|
+
for (const form of scopedNonWesternHonorifics) if (form.endsWith(".")) titleAbbrSet.add(form.replace(/[.-]+$/u, "").toLowerCase());
|
|
8039
8377
|
const exclusions = exclusionMod.default.words;
|
|
8378
|
+
const nwLocaleKeys = getScopedNonWesternLocaleKeys(languages);
|
|
8379
|
+
const [nwNameMods, nwExcludedMod] = await Promise.all([Promise.all(nwLocaleKeys.map((locale) => import(`../data/names-nw-${locale}.json`))), import("./names-nw-excluded-allcaps.mjs")]);
|
|
8380
|
+
const nonWesternNames = [];
|
|
8381
|
+
for (const mod of nwNameMods) for (const name of mod.default.names) nonWesternNames.push(name);
|
|
8382
|
+
if (dictionaries?.nonWesternNames) {
|
|
8383
|
+
const entries = languages === void 0 ? Object.entries(dictionaries.nonWesternNames) : (() => {
|
|
8384
|
+
const allowed = new Set(languages.map(normalizeCorpusLanguage));
|
|
8385
|
+
return Object.entries(dictionaries.nonWesternNames).filter(([language]) => allowed.has(normalizeCorpusLanguage(language)));
|
|
8386
|
+
})();
|
|
8387
|
+
for (const [, names] of entries) for (const name of names) nonWesternNames.push(name);
|
|
8388
|
+
}
|
|
8389
|
+
const dedupNonWestern = dedup(nonWesternNames);
|
|
8390
|
+
const dedupExcludedAllCaps = dedup(nwExcludedMod.default.words);
|
|
8040
8391
|
ctx.nameCorpus = {
|
|
8041
8392
|
firstNames: Object.freeze(new Set(dedupFirst)),
|
|
8042
8393
|
surnames: Object.freeze(new Set(dedupSurnames)),
|
|
8043
|
-
titleTokens: Object.freeze(new Set(
|
|
8394
|
+
titleTokens: Object.freeze(new Set(dedupTitles)),
|
|
8395
|
+
titleAbbreviations: Object.freeze(titleAbbrSet),
|
|
8044
8396
|
excludedWords: Object.freeze(new Set(exclusions)),
|
|
8397
|
+
nonWesternNames: Object.freeze(new Set(dedupNonWestern)),
|
|
8398
|
+
excludedAllCaps: Object.freeze(new Set(dedupExcludedAllCaps)),
|
|
8045
8399
|
firstNamesList: Object.freeze(dedupFirst),
|
|
8046
8400
|
surnamesList: Object.freeze(dedupSurnames),
|
|
8047
|
-
titlesList: Object.freeze(
|
|
8048
|
-
excludedList: Object.freeze(exclusions)
|
|
8401
|
+
titlesList: Object.freeze(dedupTitles),
|
|
8402
|
+
excludedList: Object.freeze(exclusions),
|
|
8403
|
+
nonWesternNamesList: Object.freeze(dedupNonWestern),
|
|
8404
|
+
excludedAllCapsList: Object.freeze(dedupExcludedAllCaps)
|
|
8049
8405
|
};
|
|
8050
8406
|
} catch (err) {
|
|
8051
8407
|
ctx.nameCorpusPromise = null;
|
|
@@ -8055,51 +8411,237 @@ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
|
8055
8411
|
ctx.nameCorpusPromise = promise;
|
|
8056
8412
|
return promise;
|
|
8057
8413
|
};
|
|
8058
|
-
const
|
|
8059
|
-
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
|
|
8070
|
-
|
|
8071
|
-
|
|
8072
|
-
|
|
8073
|
-
|
|
8074
|
-
|
|
8075
|
-
|
|
8076
|
-
|
|
8077
|
-
|
|
8078
|
-
|
|
8079
|
-
|
|
8080
|
-
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
|
|
8084
|
-
|
|
8085
|
-
|
|
8086
|
-
}
|
|
8087
|
-
|
|
8088
|
-
|
|
8089
|
-
|
|
8090
|
-
|
|
8091
|
-
|
|
8092
|
-
|
|
8093
|
-
|
|
8094
|
-
|
|
8095
|
-
|
|
8096
|
-
|
|
8097
|
-
|
|
8098
|
-
|
|
8099
|
-
|
|
8100
|
-
|
|
8101
|
-
|
|
8102
|
-
|
|
8414
|
+
const DECLENSION_RULES = [
|
|
8415
|
+
{
|
|
8416
|
+
ending: "",
|
|
8417
|
+
gate: /[bdfghklmnpqrstvwxzł]$/u,
|
|
8418
|
+
forms: [
|
|
8419
|
+
"a",
|
|
8420
|
+
"u",
|
|
8421
|
+
"ovi",
|
|
8422
|
+
"em",
|
|
8423
|
+
"om"
|
|
8424
|
+
]
|
|
8425
|
+
},
|
|
8426
|
+
{
|
|
8427
|
+
ending: "",
|
|
8428
|
+
gate: /[bdflmnpstvwz]$/u,
|
|
8429
|
+
forms: ["e"]
|
|
8430
|
+
},
|
|
8431
|
+
{
|
|
8432
|
+
ending: "",
|
|
8433
|
+
gate: /[cčďťňřšžjľ]$/u,
|
|
8434
|
+
forms: [
|
|
8435
|
+
"e",
|
|
8436
|
+
"i",
|
|
8437
|
+
"a",
|
|
8438
|
+
"ovi",
|
|
8439
|
+
"em",
|
|
8440
|
+
"om"
|
|
8441
|
+
]
|
|
8442
|
+
},
|
|
8443
|
+
{
|
|
8444
|
+
ending: "ek",
|
|
8445
|
+
gate: /[^aeiouyáéěíóôúůý]ek$/u,
|
|
8446
|
+
forms: [
|
|
8447
|
+
"ka",
|
|
8448
|
+
"ku",
|
|
8449
|
+
"kovi",
|
|
8450
|
+
"kem",
|
|
8451
|
+
"kom"
|
|
8452
|
+
]
|
|
8453
|
+
},
|
|
8454
|
+
{
|
|
8455
|
+
ending: "el",
|
|
8456
|
+
gate: /[^aeiouyáéěíóôúůý]el$/u,
|
|
8457
|
+
forms: [
|
|
8458
|
+
"la",
|
|
8459
|
+
"lu",
|
|
8460
|
+
"le",
|
|
8461
|
+
"lovi",
|
|
8462
|
+
"lem",
|
|
8463
|
+
"lom"
|
|
8464
|
+
]
|
|
8465
|
+
},
|
|
8466
|
+
{
|
|
8467
|
+
ending: "ec",
|
|
8468
|
+
gate: /[^aeiouyáéěíóôúůý]ec$/u,
|
|
8469
|
+
forms: [
|
|
8470
|
+
"ce",
|
|
8471
|
+
"ci",
|
|
8472
|
+
"covi",
|
|
8473
|
+
"cem",
|
|
8474
|
+
"com"
|
|
8475
|
+
]
|
|
8476
|
+
},
|
|
8477
|
+
{
|
|
8478
|
+
ending: "a",
|
|
8479
|
+
gate: /[^cčďťňřšžji]a$/u,
|
|
8480
|
+
forms: [
|
|
8481
|
+
"y",
|
|
8482
|
+
"u",
|
|
8483
|
+
"o",
|
|
8484
|
+
"ou",
|
|
8485
|
+
"ovi"
|
|
8486
|
+
]
|
|
8487
|
+
},
|
|
8488
|
+
{
|
|
8489
|
+
ending: "a",
|
|
8490
|
+
gate: /[cčďťňřšžj]a$/u,
|
|
8491
|
+
forms: [
|
|
8492
|
+
"i",
|
|
8493
|
+
"u",
|
|
8494
|
+
"o",
|
|
8495
|
+
"ou",
|
|
8496
|
+
"ovi"
|
|
8497
|
+
]
|
|
8498
|
+
},
|
|
8499
|
+
{
|
|
8500
|
+
ending: "a",
|
|
8501
|
+
gate: /ia$/u,
|
|
8502
|
+
forms: [
|
|
8503
|
+
"e",
|
|
8504
|
+
"i",
|
|
8505
|
+
"u",
|
|
8506
|
+
"ou"
|
|
8507
|
+
]
|
|
8508
|
+
},
|
|
8509
|
+
{
|
|
8510
|
+
ending: "ka",
|
|
8511
|
+
gate: /ka$/u,
|
|
8512
|
+
forms: ["ce"]
|
|
8513
|
+
},
|
|
8514
|
+
{
|
|
8515
|
+
ending: "ra",
|
|
8516
|
+
gate: /ra$/u,
|
|
8517
|
+
forms: ["ře", "re"]
|
|
8518
|
+
},
|
|
8519
|
+
{
|
|
8520
|
+
ending: "ha",
|
|
8521
|
+
gate: /ha$/u,
|
|
8522
|
+
forms: ["ze"]
|
|
8523
|
+
},
|
|
8524
|
+
{
|
|
8525
|
+
ending: "ga",
|
|
8526
|
+
gate: /ga$/u,
|
|
8527
|
+
forms: ["ze"]
|
|
8528
|
+
},
|
|
8529
|
+
{
|
|
8530
|
+
ending: "cha",
|
|
8531
|
+
gate: /cha$/u,
|
|
8532
|
+
forms: ["še"]
|
|
8533
|
+
},
|
|
8534
|
+
{
|
|
8535
|
+
ending: "a",
|
|
8536
|
+
gate: /[bdfmnptv]a$/u,
|
|
8537
|
+
forms: ["ě", "e"]
|
|
8538
|
+
},
|
|
8539
|
+
{
|
|
8540
|
+
ending: "a",
|
|
8541
|
+
gate: /[szl]a$/u,
|
|
8542
|
+
forms: ["e"]
|
|
8543
|
+
},
|
|
8544
|
+
{
|
|
8545
|
+
ending: "á",
|
|
8546
|
+
gate: /á$/u,
|
|
8547
|
+
forms: [
|
|
8548
|
+
"é",
|
|
8549
|
+
"ou",
|
|
8550
|
+
"ej",
|
|
8551
|
+
"ú"
|
|
8552
|
+
]
|
|
8553
|
+
},
|
|
8554
|
+
{
|
|
8555
|
+
ending: "ý",
|
|
8556
|
+
gate: /ý$/u,
|
|
8557
|
+
forms: [
|
|
8558
|
+
"ého",
|
|
8559
|
+
"ému",
|
|
8560
|
+
"ém",
|
|
8561
|
+
"ým"
|
|
8562
|
+
]
|
|
8563
|
+
},
|
|
8564
|
+
{
|
|
8565
|
+
ending: "",
|
|
8566
|
+
gate: /[íiy]$/u,
|
|
8567
|
+
forms: [
|
|
8568
|
+
"ho",
|
|
8569
|
+
"mu",
|
|
8570
|
+
"m"
|
|
8571
|
+
]
|
|
8572
|
+
},
|
|
8573
|
+
{
|
|
8574
|
+
ending: "e",
|
|
8575
|
+
gate: /[^aeouyáéěíóôúůý]e$/u,
|
|
8576
|
+
forms: ["i", "í"]
|
|
8577
|
+
},
|
|
8578
|
+
{
|
|
8579
|
+
ending: "o",
|
|
8580
|
+
gate: /o$/u,
|
|
8581
|
+
forms: [
|
|
8582
|
+
"a",
|
|
8583
|
+
"ovi",
|
|
8584
|
+
"em",
|
|
8585
|
+
"om"
|
|
8586
|
+
]
|
|
8587
|
+
}
|
|
8588
|
+
];
|
|
8589
|
+
/**
|
|
8590
|
+
* Generate declined Czech/Slovak variants of a
|
|
8591
|
+
* nominative name. Only variants licensed by the
|
|
8592
|
+
* ending-shape rules are produced, which bounds the
|
|
8593
|
+
* deny-list AC pattern growth. Returns an empty array
|
|
8594
|
+
* for names too short to decline safely.
|
|
8595
|
+
*/
|
|
8596
|
+
const expandNameDeclensions = (name) => {
|
|
8597
|
+
if (name.length < 3) return [];
|
|
8598
|
+
const lower = name.toLowerCase();
|
|
8599
|
+
const variants = [];
|
|
8600
|
+
for (const rule of DECLENSION_RULES) {
|
|
8601
|
+
if (!rule.gate.test(lower)) continue;
|
|
8602
|
+
const stem = name.slice(0, name.length - rule.ending.length);
|
|
8603
|
+
if (stem.length < 2) continue;
|
|
8604
|
+
for (const form of rule.forms) variants.push(stem + form);
|
|
8605
|
+
}
|
|
8606
|
+
return variants;
|
|
8607
|
+
};
|
|
8608
|
+
/**
|
|
8609
|
+
* Strip Czech/Slovak case endings from a token using
|
|
8610
|
+
* the inverse of DECLENSION_RULES. Returns candidate
|
|
8611
|
+
* nominative forms; each candidate is validated against
|
|
8612
|
+
* the rule's shape gate, so only bases the paradigm
|
|
8613
|
+
* could actually have declined are proposed. Callers
|
|
8614
|
+
* check candidates against the name corpus.
|
|
8615
|
+
*/
|
|
8616
|
+
const stripInflection = (token) => {
|
|
8617
|
+
const candidates = [];
|
|
8618
|
+
for (const rule of DECLENSION_RULES) for (const form of rule.forms) {
|
|
8619
|
+
if (token.length <= form.length + 2 || !token.endsWith(form)) continue;
|
|
8620
|
+
const base = token.slice(0, token.length - form.length) + rule.ending;
|
|
8621
|
+
if (!/^\p{Lu}/u.test(base)) continue;
|
|
8622
|
+
if (!rule.gate.test(base.toLowerCase())) continue;
|
|
8623
|
+
candidates.push(base);
|
|
8624
|
+
}
|
|
8625
|
+
return candidates;
|
|
8626
|
+
};
|
|
8627
|
+
const TOKEN_TYPE = {
|
|
8628
|
+
NAME: "name",
|
|
8629
|
+
SURNAME: "surname",
|
|
8630
|
+
TITLE: "title",
|
|
8631
|
+
ABBREVIATION: "abbreviation",
|
|
8632
|
+
JA_SUFFIX: "ja_suffix",
|
|
8633
|
+
ARABIC_CONNECTOR: "arabic_connector",
|
|
8634
|
+
CAPITALIZED: "capitalized",
|
|
8635
|
+
OTHER: "other"
|
|
8636
|
+
};
|
|
8637
|
+
const PERSON_CHAIN_BREAK_RE$1 = /[!?;:]/u;
|
|
8638
|
+
const isInitialContinuationGap$1 = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
8639
|
+
/**
|
|
8640
|
+
* Check if a token is in the first-name set, either
|
|
8641
|
+
* directly or after stripping Czech/Slovak inflection.
|
|
8642
|
+
*/
|
|
8643
|
+
const isFirstNameToken = (token, corpus) => {
|
|
8644
|
+
if (corpus.firstNames.has(token)) return true;
|
|
8103
8645
|
return stripInflection(token).some((b) => corpus.firstNames.has(b));
|
|
8104
8646
|
};
|
|
8105
8647
|
/**
|
|
@@ -8115,6 +8657,141 @@ const isSurnameToken = (token, corpus) => {
|
|
|
8115
8657
|
* abbreviation: "J.", "M.", etc.
|
|
8116
8658
|
*/
|
|
8117
8659
|
const isAbbreviation = (token) => token.length === 2 && /^\p{Lu}$/u.test(token[0] ?? "") && token[1] === ".";
|
|
8660
|
+
/**
|
|
8661
|
+
* Title-case a token, re-capitalizing after apostrophes
|
|
8662
|
+
* so D'Souza, O'Brien, Hon'ble match the corpus.
|
|
8663
|
+
*/
|
|
8664
|
+
const titleCaseWithApostrophe = (text) => (text[0]?.toUpperCase() ?? "") + text.slice(1).toLowerCase().replace(/'\p{Ll}/gu, (m) => m.toUpperCase());
|
|
8665
|
+
/**
|
|
8666
|
+
* Check if a token is in the non-Western name corpus.
|
|
8667
|
+
* Tries both title-cased (with apostrophe re-cap) and
|
|
8668
|
+
* raw forms.
|
|
8669
|
+
*/
|
|
8670
|
+
const isNonWesternNameToken = (token, corpus) => {
|
|
8671
|
+
if (corpus.nonWesternNames.has(token)) return true;
|
|
8672
|
+
return corpus.nonWesternNames.has(titleCaseWithApostrophe(token));
|
|
8673
|
+
};
|
|
8674
|
+
/** Japanese honorific suffixes attached via hyphen. */
|
|
8675
|
+
const JA_SUFFIXES = new Set([
|
|
8676
|
+
"san",
|
|
8677
|
+
"sama",
|
|
8678
|
+
"sensei"
|
|
8679
|
+
]);
|
|
8680
|
+
/** Arabic patronymic connectors. */
|
|
8681
|
+
const ARABIC_CONNECTORS = new Set([
|
|
8682
|
+
"bin",
|
|
8683
|
+
"bint",
|
|
8684
|
+
"ibn",
|
|
8685
|
+
"al",
|
|
8686
|
+
"el"
|
|
8687
|
+
]);
|
|
8688
|
+
/** Al-/El- prefixed name pattern (e.g., Al-Rashid, El-Amin). */
|
|
8689
|
+
const AL_EL_PREFIX_RE = /^[Aa]l-[A-Z][a-z]+$|^[Ee]l-[A-Z][a-z]+$/u;
|
|
8690
|
+
/** CJK name detection: 2-4 Han chars not adjacent to others. */
|
|
8691
|
+
const CJK_NAME_RE = /(?<!\p{Script=Han})\p{Script=Han}{2,4}(?!\p{Script=Han})/gu;
|
|
8692
|
+
/** Han character threshold for CJK-majority documents. */
|
|
8693
|
+
const CJK_HAN_RATIO = .15;
|
|
8694
|
+
const CJK_NON_PERSON_TERMS = new Set([
|
|
8695
|
+
"中国",
|
|
8696
|
+
"中國",
|
|
8697
|
+
"中文",
|
|
8698
|
+
"人民",
|
|
8699
|
+
"公司",
|
|
8700
|
+
"香港",
|
|
8701
|
+
"台湾",
|
|
8702
|
+
"臺灣",
|
|
8703
|
+
"日本",
|
|
8704
|
+
"韩国",
|
|
8705
|
+
"韓國"
|
|
8706
|
+
]);
|
|
8707
|
+
const CJK_SURNAME_CHARS = new Set([
|
|
8708
|
+
"王",
|
|
8709
|
+
"李",
|
|
8710
|
+
"张",
|
|
8711
|
+
"張",
|
|
8712
|
+
"刘",
|
|
8713
|
+
"劉",
|
|
8714
|
+
"陈",
|
|
8715
|
+
"陳",
|
|
8716
|
+
"杨",
|
|
8717
|
+
"楊",
|
|
8718
|
+
"黄",
|
|
8719
|
+
"黃",
|
|
8720
|
+
"赵",
|
|
8721
|
+
"趙",
|
|
8722
|
+
"吴",
|
|
8723
|
+
"吳",
|
|
8724
|
+
"周",
|
|
8725
|
+
"徐",
|
|
8726
|
+
"孙",
|
|
8727
|
+
"孫",
|
|
8728
|
+
"马",
|
|
8729
|
+
"馬",
|
|
8730
|
+
"朱",
|
|
8731
|
+
"胡",
|
|
8732
|
+
"郭",
|
|
8733
|
+
"何",
|
|
8734
|
+
"林",
|
|
8735
|
+
"高",
|
|
8736
|
+
"梁",
|
|
8737
|
+
"郑",
|
|
8738
|
+
"鄭",
|
|
8739
|
+
"罗",
|
|
8740
|
+
"羅",
|
|
8741
|
+
"宋",
|
|
8742
|
+
"谢",
|
|
8743
|
+
"謝",
|
|
8744
|
+
"唐",
|
|
8745
|
+
"韩",
|
|
8746
|
+
"韓",
|
|
8747
|
+
"曹",
|
|
8748
|
+
"许",
|
|
8749
|
+
"許",
|
|
8750
|
+
"邓",
|
|
8751
|
+
"鄧",
|
|
8752
|
+
"萧",
|
|
8753
|
+
"蕭",
|
|
8754
|
+
"田",
|
|
8755
|
+
"山",
|
|
8756
|
+
"佐",
|
|
8757
|
+
"鈴",
|
|
8758
|
+
"渡",
|
|
8759
|
+
"伊",
|
|
8760
|
+
"中",
|
|
8761
|
+
"小",
|
|
8762
|
+
"吉",
|
|
8763
|
+
"金",
|
|
8764
|
+
"朴",
|
|
8765
|
+
"박",
|
|
8766
|
+
"김",
|
|
8767
|
+
"이",
|
|
8768
|
+
"최",
|
|
8769
|
+
"정",
|
|
8770
|
+
"강",
|
|
8771
|
+
"조",
|
|
8772
|
+
"윤",
|
|
8773
|
+
"장",
|
|
8774
|
+
"임",
|
|
8775
|
+
"한"
|
|
8776
|
+
].join(""));
|
|
8777
|
+
const isLikelyCjkPersonName = (text) => {
|
|
8778
|
+
if (CJK_NON_PERSON_TERMS.has(text)) return false;
|
|
8779
|
+
const first = text.at(0);
|
|
8780
|
+
return first !== void 0 && CJK_SURNAME_CHARS.has(first);
|
|
8781
|
+
};
|
|
8782
|
+
/** Organization keyword filter. */
|
|
8783
|
+
const ORG_WORDS = /\b(?:Group|Company|LLC|LLP|LP|Inc|Ltd|Corp|Corporation|Holdings|Partners|Association|University|Bank|Fund|Trust|Agency|Government|Ministry|Office|Department|Council|Board|Committee|Commission|Services|Solutions|Technologies|Systems|Analytics|Software)\b/i;
|
|
8784
|
+
const isOrganization = (text) => ORG_WORDS.test(text);
|
|
8785
|
+
/** Deduplicate overlapping entity spans (first wins). */
|
|
8786
|
+
const deduplicateSpans$1 = (entities) => {
|
|
8787
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start || b.end - a.end);
|
|
8788
|
+
const result = [];
|
|
8789
|
+
for (const entity of sorted) {
|
|
8790
|
+
const last = result.at(-1);
|
|
8791
|
+
if (!last || entity.start >= last.end) result.push(entity);
|
|
8792
|
+
}
|
|
8793
|
+
return result;
|
|
8794
|
+
};
|
|
8118
8795
|
const segmenter$1 = new Intl.Segmenter(void 0, { granularity: "word" });
|
|
8119
8796
|
/**
|
|
8120
8797
|
* Split text into word segments using Intl.Segmenter.
|
|
@@ -8163,21 +8840,55 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8163
8840
|
text,
|
|
8164
8841
|
type: TOKEN_TYPE.TITLE,
|
|
8165
8842
|
start,
|
|
8843
|
+
end,
|
|
8844
|
+
...corpus.titleAbbreviations.has(stripped) ? { titleAbbreviation: true } : {}
|
|
8845
|
+
};
|
|
8846
|
+
if (JA_SUFFIXES.has(lower)) return {
|
|
8847
|
+
text,
|
|
8848
|
+
type: TOKEN_TYPE.JA_SUFFIX,
|
|
8849
|
+
start,
|
|
8166
8850
|
end
|
|
8167
8851
|
};
|
|
8852
|
+
if (ARABIC_CONNECTORS.has(lower)) return {
|
|
8853
|
+
text,
|
|
8854
|
+
type: TOKEN_TYPE.ARABIC_CONNECTOR,
|
|
8855
|
+
start,
|
|
8856
|
+
end
|
|
8857
|
+
};
|
|
8858
|
+
if (AL_EL_PREFIX_RE.test(text)) return {
|
|
8859
|
+
text,
|
|
8860
|
+
type: TOKEN_TYPE.NAME,
|
|
8861
|
+
start,
|
|
8862
|
+
end,
|
|
8863
|
+
nonWestern: true
|
|
8864
|
+
};
|
|
8168
8865
|
if (isAbbreviation(text)) return {
|
|
8169
8866
|
text,
|
|
8170
8867
|
type: TOKEN_TYPE.ABBREVIATION,
|
|
8171
8868
|
start,
|
|
8172
8869
|
end
|
|
8173
8870
|
};
|
|
8871
|
+
if (/^(?:\p{Lu}\.)+\p{Lu}?$/u.test(text)) return {
|
|
8872
|
+
text,
|
|
8873
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
8874
|
+
start,
|
|
8875
|
+
end
|
|
8876
|
+
};
|
|
8174
8877
|
if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
|
|
8175
8878
|
const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
|
|
8176
8879
|
const before = fullText.slice(lineStart, start).trimEnd();
|
|
8177
8880
|
const lastWord = /\p{L}[\p{L}\p{M}'-]*$/u.exec(before)?.[0];
|
|
8178
|
-
|
|
8179
|
-
|
|
8180
|
-
|
|
8881
|
+
const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus) || isNonWesternNameToken(token, corpus);
|
|
8882
|
+
if (lastWord && lookup(lastWord)) return {
|
|
8883
|
+
text,
|
|
8884
|
+
type: TOKEN_TYPE.ABBREVIATION,
|
|
8885
|
+
start,
|
|
8886
|
+
end
|
|
8887
|
+
};
|
|
8888
|
+
const afterDot = fullText.slice(end + 1).trimStart();
|
|
8889
|
+
const nextWord = /^\p{L}[\p{L}\p{M}'-]*/u.exec(afterDot)?.[0];
|
|
8890
|
+
if (nextWord) {
|
|
8891
|
+
if (lookup(nextWord) || nextWord.length === 1 && UPPER_START_RE.test(nextWord)) return {
|
|
8181
8892
|
text,
|
|
8182
8893
|
type: TOKEN_TYPE.ABBREVIATION,
|
|
8183
8894
|
start,
|
|
@@ -8197,26 +8908,55 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8197
8908
|
start,
|
|
8198
8909
|
end
|
|
8199
8910
|
};
|
|
8200
|
-
if (text.length <
|
|
8911
|
+
if (text.length < 2) return {
|
|
8912
|
+
text,
|
|
8913
|
+
type: TOKEN_TYPE.OTHER,
|
|
8914
|
+
start,
|
|
8915
|
+
end
|
|
8916
|
+
};
|
|
8917
|
+
if (text.length < 3 && !isNonWesternNameToken(text, corpus) && !JA_SUFFIXES.has(lower) && !ARABIC_CONNECTORS.has(lower) && !(ALL_UPPER_RE.test(text) && !corpus.excludedAllCaps.has(text))) return {
|
|
8201
8918
|
text,
|
|
8202
8919
|
type: TOKEN_TYPE.OTHER,
|
|
8203
8920
|
start,
|
|
8204
8921
|
end
|
|
8205
8922
|
};
|
|
8206
8923
|
if (text.length >= 3 && ALL_UPPER_RE.test(text)) {
|
|
8924
|
+
if (corpus.excludedAllCaps.has(text)) return {
|
|
8925
|
+
text,
|
|
8926
|
+
type: TOKEN_TYPE.OTHER,
|
|
8927
|
+
start,
|
|
8928
|
+
end
|
|
8929
|
+
};
|
|
8930
|
+
const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
|
|
8931
|
+
const nwMatch = isNonWesternNameToken(titleCased, corpus);
|
|
8932
|
+
if (nwMatch && !isFirstNameToken(titleCased, corpus)) return {
|
|
8933
|
+
text,
|
|
8934
|
+
type: TOKEN_TYPE.NAME,
|
|
8935
|
+
start,
|
|
8936
|
+
end,
|
|
8937
|
+
nonWestern: true
|
|
8938
|
+
};
|
|
8207
8939
|
if (isAllCapsContextLine(fullText, start) && isAllCapsLineNameShaped(fullText, start)) {
|
|
8208
|
-
const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
|
|
8209
8940
|
if (isFirstNameToken(titleCased, corpus)) return {
|
|
8210
8941
|
text,
|
|
8211
8942
|
type: TOKEN_TYPE.NAME,
|
|
8212
8943
|
start,
|
|
8213
|
-
end
|
|
8944
|
+
end,
|
|
8945
|
+
...nwMatch ? { nonWestern: true } : {}
|
|
8214
8946
|
};
|
|
8215
8947
|
if (isSurnameToken(titleCased, corpus)) return {
|
|
8216
8948
|
text,
|
|
8217
8949
|
type: TOKEN_TYPE.SURNAME,
|
|
8218
8950
|
start,
|
|
8219
|
-
end
|
|
8951
|
+
end,
|
|
8952
|
+
...nwMatch ? { nonWestern: true } : {}
|
|
8953
|
+
};
|
|
8954
|
+
if (nwMatch) return {
|
|
8955
|
+
text,
|
|
8956
|
+
type: TOKEN_TYPE.NAME,
|
|
8957
|
+
start,
|
|
8958
|
+
end,
|
|
8959
|
+
nonWestern: true
|
|
8220
8960
|
};
|
|
8221
8961
|
}
|
|
8222
8962
|
return {
|
|
@@ -8232,17 +8972,32 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8232
8972
|
start,
|
|
8233
8973
|
end
|
|
8234
8974
|
};
|
|
8235
|
-
if (isFirstNameToken(text, corpus))
|
|
8975
|
+
if (isFirstNameToken(text, corpus)) {
|
|
8976
|
+
const nw = isNonWesternNameToken(text, corpus);
|
|
8977
|
+
return {
|
|
8978
|
+
text,
|
|
8979
|
+
type: TOKEN_TYPE.NAME,
|
|
8980
|
+
start,
|
|
8981
|
+
end,
|
|
8982
|
+
...nw ? { nonWestern: true } : {}
|
|
8983
|
+
};
|
|
8984
|
+
}
|
|
8985
|
+
if (isSurnameToken(text, corpus)) {
|
|
8986
|
+
const nw = isNonWesternNameToken(text, corpus);
|
|
8987
|
+
return {
|
|
8988
|
+
text,
|
|
8989
|
+
type: TOKEN_TYPE.SURNAME,
|
|
8990
|
+
start,
|
|
8991
|
+
end,
|
|
8992
|
+
...nw ? { nonWestern: true } : {}
|
|
8993
|
+
};
|
|
8994
|
+
}
|
|
8995
|
+
if (isNonWesternNameToken(text, corpus)) return {
|
|
8236
8996
|
text,
|
|
8237
8997
|
type: TOKEN_TYPE.NAME,
|
|
8238
8998
|
start,
|
|
8239
|
-
end
|
|
8240
|
-
|
|
8241
|
-
if (isSurnameToken(text, corpus)) return {
|
|
8242
|
-
text,
|
|
8243
|
-
type: TOKEN_TYPE.SURNAME,
|
|
8244
|
-
start,
|
|
8245
|
-
end
|
|
8999
|
+
end,
|
|
9000
|
+
nonWestern: true
|
|
8246
9001
|
};
|
|
8247
9002
|
return {
|
|
8248
9003
|
text,
|
|
@@ -8254,11 +9009,12 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8254
9009
|
/**
|
|
8255
9010
|
* Detect person names by looking up tokens against the
|
|
8256
9011
|
* name corpus, then chaining adjacent name-like tokens.
|
|
9012
|
+
* Handles both Western and non-Western name patterns.
|
|
8257
9013
|
*
|
|
8258
9014
|
* Requires initNameCorpus() to have been called first.
|
|
8259
9015
|
* If not initialized, returns an empty array.
|
|
8260
9016
|
*
|
|
8261
|
-
* Scoring:
|
|
9017
|
+
* Scoring (Western):
|
|
8262
9018
|
* TITLE + NAME/SURNAME → 0.95
|
|
8263
9019
|
* NAME + NAME/SURNAME → 0.9
|
|
8264
9020
|
* SURNAME + NAME/SURNAME → 0.9
|
|
@@ -8266,18 +9022,63 @@ const classifyToken = (word, corpus, fullText) => {
|
|
|
8266
9022
|
* ABBREVIATION + NAME → 0.7
|
|
8267
9023
|
* Standalone NAME → 0.5 (low confidence)
|
|
8268
9024
|
* Standalone SURNAME → skip (too ambiguous)
|
|
8269
|
-
|
|
8270
|
-
|
|
9025
|
+
*
|
|
9026
|
+
* Scoring (non-Western, when chain contains nonWestern tokens):
|
|
9027
|
+
* TITLE + (nonWestern|CAPITALIZED) → 0.95
|
|
9028
|
+
* JA_SUFFIX + (CAPITALIZED|nonWestern) → 0.9
|
|
9029
|
+
* ARABIC_CONNECTOR + nonWestern → 0.9
|
|
9030
|
+
* 2+ nonWestern tokens → 0.9
|
|
9031
|
+
* nonWestern + (CAPITALIZED|ABBREVIATION) → 0.9
|
|
9032
|
+
* Standalone nonWestern mid-sentence → 0.5
|
|
9033
|
+
*/
|
|
9034
|
+
const detectNameCorpus = (fullText, ctx = defaultContext, options = {}) => {
|
|
8271
9035
|
const corpus = getCorpus(ctx);
|
|
8272
9036
|
if (!corpus) return [];
|
|
8273
|
-
const
|
|
9037
|
+
const supplementalMode = options.mode === "supplemental";
|
|
8274
9038
|
const entities = [];
|
|
9039
|
+
const threshold = Math.ceil(fullText.length * CJK_HAN_RATIO);
|
|
9040
|
+
let hanCount = 0;
|
|
9041
|
+
for (const _ of fullText.matchAll(/\p{Script=Han}/gu)) {
|
|
9042
|
+
hanCount++;
|
|
9043
|
+
if (hanCount >= threshold) break;
|
|
9044
|
+
}
|
|
9045
|
+
if (fullText.length < 100 || hanCount < threshold) {
|
|
9046
|
+
CJK_NAME_RE.lastIndex = 0;
|
|
9047
|
+
let match;
|
|
9048
|
+
while ((match = CJK_NAME_RE.exec(fullText)) !== null) {
|
|
9049
|
+
const cjkText = match[0];
|
|
9050
|
+
if (isLikelyCjkPersonName(cjkText) && !isOrganization(cjkText)) entities.push({
|
|
9051
|
+
start: match.index,
|
|
9052
|
+
end: match.index + cjkText.length,
|
|
9053
|
+
label: "person",
|
|
9054
|
+
text: cjkText,
|
|
9055
|
+
score: .95,
|
|
9056
|
+
source: DETECTION_SOURCES.REGEX
|
|
9057
|
+
});
|
|
9058
|
+
}
|
|
9059
|
+
}
|
|
9060
|
+
const words = segmentWords(fullText);
|
|
9061
|
+
const tokens = [];
|
|
9062
|
+
for (let idx = 0; idx < words.length; idx++) {
|
|
9063
|
+
const word = words[idx];
|
|
9064
|
+
const lower = word.text.toLowerCase();
|
|
9065
|
+
const nextWord = words[idx + 1];
|
|
9066
|
+
if ((lower === "s" || lower === "d" || lower === "w" || lower === "r") && fullText[word.end] === "/" && nextWord && nextWord.start === word.end + 1 && nextWord.text.toLowerCase() === "o") {
|
|
9067
|
+
tokens.push({
|
|
9068
|
+
text: fullText.slice(word.start, word.end + 2),
|
|
9069
|
+
type: TOKEN_TYPE.ARABIC_CONNECTOR,
|
|
9070
|
+
start: word.start,
|
|
9071
|
+
end: word.end + 2
|
|
9072
|
+
});
|
|
9073
|
+
idx++;
|
|
9074
|
+
} else tokens.push(classifyToken(word, corpus, fullText));
|
|
9075
|
+
}
|
|
8275
9076
|
const consumed = /* @__PURE__ */ new Set();
|
|
8276
9077
|
for (let i = 0; i < tokens.length; i++) {
|
|
8277
9078
|
if (consumed.has(i)) continue;
|
|
8278
9079
|
const token = tokens[i];
|
|
8279
9080
|
if (!token) continue;
|
|
8280
|
-
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION) continue;
|
|
9081
|
+
if (token.type !== TOKEN_TYPE.TITLE && token.type !== TOKEN_TYPE.NAME && token.type !== TOKEN_TYPE.SURNAME && token.type !== TOKEN_TYPE.ABBREVIATION && token.type !== TOKEN_TYPE.ARABIC_CONNECTOR) continue;
|
|
8281
9082
|
const MAX_CHAIN = 5;
|
|
8282
9083
|
const chain = [token];
|
|
8283
9084
|
let j = i + 1;
|
|
@@ -8287,10 +9088,12 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8287
9088
|
const prev = chain.at(-1);
|
|
8288
9089
|
if (prev) {
|
|
8289
9090
|
const gap = fullText.slice(prev.end, next.start);
|
|
8290
|
-
const
|
|
9091
|
+
const periodIsPartOfPrevToken = prev.type === TOKEN_TYPE.ABBREVIATION || prev.type === TOKEN_TYPE.TITLE && prev.titleAbbreviation === true;
|
|
9092
|
+
const breaksOnPeriod = gap.includes(".") && !isInitialContinuationGap$1(prev.text, gap) && !periodIsPartOfPrevToken;
|
|
8291
9093
|
if (gap.includes("\n") || PERSON_CHAIN_BREAK_RE$1.test(gap) || breaksOnPeriod) break;
|
|
9094
|
+
if (next.type === TOKEN_TYPE.JA_SUFFIX && gap !== "-" && gap.trim() !== "") break;
|
|
8292
9095
|
}
|
|
8293
|
-
if (next.type
|
|
9096
|
+
if (next.type !== TOKEN_TYPE.OTHER) {
|
|
8294
9097
|
chain.push(next);
|
|
8295
9098
|
j++;
|
|
8296
9099
|
} else break;
|
|
@@ -8299,10 +9102,24 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8299
9102
|
const hasCorpusName = chain.some((t) => isCorpusMatch(t.type));
|
|
8300
9103
|
const hasFirstName = chain.some((t) => t.type === TOKEN_TYPE.NAME);
|
|
8301
9104
|
const hasAbbreviation = chain.some((t) => t.type === TOKEN_TYPE.ABBREVIATION);
|
|
9105
|
+
const hasNonWestern = chain.some((t) => t.nonWestern === true);
|
|
9106
|
+
const hasJaSuffix = chain.some((t) => t.type === TOKEN_TYPE.JA_SUFFIX);
|
|
9107
|
+
const hasArabicConnector = chain.some((t) => t.type === TOKEN_TYPE.ARABIC_CONNECTOR);
|
|
8302
9108
|
const corpusCount = chain.filter((t) => isCorpusMatch(t.type)).length;
|
|
8303
9109
|
const capitalizedCount = chain.filter((t) => t.type === TOKEN_TYPE.CAPITALIZED).length;
|
|
9110
|
+
const nonWesternCount = chain.filter((t) => t.nonWestern).length;
|
|
8304
9111
|
let score;
|
|
8305
|
-
if (
|
|
9112
|
+
if (hasNonWestern) {
|
|
9113
|
+
if (hasTitle && (nonWesternCount > 0 || capitalizedCount > 0)) score = .95;
|
|
9114
|
+
else if (hasJaSuffix && (capitalizedCount > 0 || nonWesternCount > 0)) score = .9;
|
|
9115
|
+
else if (hasArabicConnector && nonWesternCount > 0) score = .9;
|
|
9116
|
+
else if (nonWesternCount >= 2) score = .9;
|
|
9117
|
+
else if (nonWesternCount > 0 && (capitalizedCount > 0 || hasAbbreviation)) score = .9;
|
|
9118
|
+
else if (nonWesternCount === 1 && chain.length === 1 && !isSentenceStart(fullText, token.start)) score = .5;
|
|
9119
|
+
else continue;
|
|
9120
|
+
if (supplementalMode && score < .9) continue;
|
|
9121
|
+
} else if (supplementalMode) continue;
|
|
9122
|
+
else if (hasTitle && hasCorpusName) score = .95;
|
|
8306
9123
|
else if (corpusCount >= 2) score = .9;
|
|
8307
9124
|
else if (hasCorpusName && capitalizedCount > 0) score = .7;
|
|
8308
9125
|
else if (hasAbbreviation && hasCorpusName) score = .7;
|
|
@@ -8313,7 +9130,10 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8313
9130
|
score = .5;
|
|
8314
9131
|
} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
|
|
8315
9132
|
else if (hasTitle && chain.length === 1) continue;
|
|
8316
|
-
else {
|
|
9133
|
+
else if (hasJaSuffix || hasArabicConnector) {
|
|
9134
|
+
if (!hasCorpusName && !hasFirstName) continue;
|
|
9135
|
+
score = .5;
|
|
9136
|
+
} else {
|
|
8317
9137
|
if (!hasCorpusName) continue;
|
|
8318
9138
|
score = .5;
|
|
8319
9139
|
}
|
|
@@ -8323,6 +9143,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8323
9143
|
const start = first.start;
|
|
8324
9144
|
const end = last.end;
|
|
8325
9145
|
const text = fullText.slice(start, end);
|
|
9146
|
+
if (isOrganization(text)) continue;
|
|
8326
9147
|
for (let k = i; k < i + chain.length; k++) consumed.add(k);
|
|
8327
9148
|
entities.push({
|
|
8328
9149
|
start,
|
|
@@ -8333,14 +9154,14 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8333
9154
|
source: DETECTION_SOURCES.REGEX
|
|
8334
9155
|
});
|
|
8335
9156
|
}
|
|
8336
|
-
return entities;
|
|
9157
|
+
return deduplicateSpans$1(entities);
|
|
8337
9158
|
};
|
|
8338
9159
|
//#endregion
|
|
8339
9160
|
//#region src/detectors/signatures.ts
|
|
8340
9161
|
const SLASH_S_RE = /\/s\/[ \t]*/g;
|
|
8341
9162
|
const LABELLED_NAME_RE = /\b(?:By|Name)[ \t]*:[ \t]*(?:\/s\/[ \t]+)?([^\n]*?)(?=\s{3,}|[\t]|\n|$)/gi;
|
|
8342
9163
|
const WITNESS_ANCHOR_RE = /\bIN WITNESS WHEREOF\b[^\n]*\n/gi;
|
|
8343
|
-
const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
|
|
9164
|
+
const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|da|das|do|dos|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
|
|
8344
9165
|
const CAP_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'\\-]{0,30}";
|
|
8345
9166
|
const NAME_SHAPE_RE = new RegExp(`^${CAP_TOKEN}(?:[ \\t]+(?:${NAME_PARTICLE}|${CAP_TOKEN})){1,4}$`, "u");
|
|
8346
9167
|
const MAX_NAME_LEN = 60;
|
|
@@ -8445,219 +9266,6 @@ const detectSignatures = (fullText, _ctx = defaultContext) => {
|
|
|
8445
9266
|
return results;
|
|
8446
9267
|
};
|
|
8447
9268
|
//#endregion
|
|
8448
|
-
//#region src/config/titles.ts
|
|
8449
|
-
/**
|
|
8450
|
-
* Academic and professional title prefixes.
|
|
8451
|
-
* Plain text; the detector auto-escapes for regex.
|
|
8452
|
-
* Sorted longest-first at build time.
|
|
8453
|
-
*/
|
|
8454
|
-
const TITLE_PREFIXES = [
|
|
8455
|
-
"Ing.",
|
|
8456
|
-
"Mgr.",
|
|
8457
|
-
"MgA.",
|
|
8458
|
-
"Bc.",
|
|
8459
|
-
"BcA.",
|
|
8460
|
-
"JUDr.",
|
|
8461
|
-
"MUDr.",
|
|
8462
|
-
"MVDr.",
|
|
8463
|
-
"MDDr.",
|
|
8464
|
-
"PhDr.",
|
|
8465
|
-
"RNDr.",
|
|
8466
|
-
"PaedDr.",
|
|
8467
|
-
"ThDr.",
|
|
8468
|
-
"ThLic.",
|
|
8469
|
-
"ICDr.",
|
|
8470
|
-
"RSDr.",
|
|
8471
|
-
"PharmDr.",
|
|
8472
|
-
"artD.",
|
|
8473
|
-
"akad.",
|
|
8474
|
-
"doc.",
|
|
8475
|
-
"prof.",
|
|
8476
|
-
"ao. Univ.-Prof.",
|
|
8477
|
-
"o. Univ.-Prof.",
|
|
8478
|
-
"Univ.-Prof.",
|
|
8479
|
-
"Hon.-Prof.",
|
|
8480
|
-
"em. Prof.",
|
|
8481
|
-
"Dr. med. dent.",
|
|
8482
|
-
"Dr. med. vet.",
|
|
8483
|
-
"Dr. med.",
|
|
8484
|
-
"Dr. rer. nat.",
|
|
8485
|
-
"Dr. rer. soc.",
|
|
8486
|
-
"Dr. rer. pol.",
|
|
8487
|
-
"Dr. sc. tech.",
|
|
8488
|
-
"Dr. sc. nat.",
|
|
8489
|
-
"Dr. sc. hum.",
|
|
8490
|
-
"Dr. iur.",
|
|
8491
|
-
"Dr. jur.",
|
|
8492
|
-
"Dr. theol.",
|
|
8493
|
-
"Dr. oec.",
|
|
8494
|
-
"Dr. techn.",
|
|
8495
|
-
"Dr. h. c.",
|
|
8496
|
-
"Dr. phil.",
|
|
8497
|
-
"Dr.-Ing.",
|
|
8498
|
-
"Dr. Ing.",
|
|
8499
|
-
"Dr.",
|
|
8500
|
-
"Dipl.-Wirt.-Ing.",
|
|
8501
|
-
"Dipl.-Betriebsw.",
|
|
8502
|
-
"Dipl.-Inform.",
|
|
8503
|
-
"Dipl.-Volksw.",
|
|
8504
|
-
"Dipl.-Psych.",
|
|
8505
|
-
"Dipl.-Phys.",
|
|
8506
|
-
"Dipl.-Chem.",
|
|
8507
|
-
"Dipl.-Biol.",
|
|
8508
|
-
"Dipl.-Math.",
|
|
8509
|
-
"Dipl.-Päd.",
|
|
8510
|
-
"Dipl.-Soz.",
|
|
8511
|
-
"Dipl.-Kfm.",
|
|
8512
|
-
"Dipl.-Jur.",
|
|
8513
|
-
"Dipl. Ing.",
|
|
8514
|
-
"Dipl.-Ing.",
|
|
8515
|
-
"Mag. rer. soc. oec.",
|
|
8516
|
-
"Mag. rer. nat.",
|
|
8517
|
-
"Mag. phil.",
|
|
8518
|
-
"Mag. iur.",
|
|
8519
|
-
"Mag. arch.",
|
|
8520
|
-
"Mag. pharm.",
|
|
8521
|
-
"Mag. (FH)",
|
|
8522
|
-
"Mag.",
|
|
8523
|
-
"Bakk. rer. nat.",
|
|
8524
|
-
"Bakk. techn.",
|
|
8525
|
-
"Bakk. phil.",
|
|
8526
|
-
"Bakk.",
|
|
8527
|
-
"Lic. phil.",
|
|
8528
|
-
"Lic. iur.",
|
|
8529
|
-
"Lic. oec.",
|
|
8530
|
-
"Lic. theol.",
|
|
8531
|
-
"Lic.",
|
|
8532
|
-
"Priv.-Doz.",
|
|
8533
|
-
"PD",
|
|
8534
|
-
"RA"
|
|
8535
|
-
];
|
|
8536
|
-
/**
|
|
8537
|
-
* Courtesy/honorific titles that precede a person's
|
|
8538
|
-
* name. Sorted alphabetically. The detector escapes
|
|
8539
|
-
* dots and adds \b for entries in HONORIFIC_BOUNDARY.
|
|
8540
|
-
*/
|
|
8541
|
-
const HONORIFICS = [
|
|
8542
|
-
"Avv.",
|
|
8543
|
-
"Dame",
|
|
8544
|
-
"Doamna",
|
|
8545
|
-
"Domnul",
|
|
8546
|
-
"Don",
|
|
8547
|
-
"Doña",
|
|
8548
|
-
"Dott.",
|
|
8549
|
-
"Judge",
|
|
8550
|
-
"Justice",
|
|
8551
|
-
"Lady",
|
|
8552
|
-
"Lord",
|
|
8553
|
-
"M.",
|
|
8554
|
-
"Madame",
|
|
8555
|
-
"Mademoiselle",
|
|
8556
|
-
"Maître",
|
|
8557
|
-
"Me",
|
|
8558
|
-
"Messrs",
|
|
8559
|
-
"Miss",
|
|
8560
|
-
"Mlle",
|
|
8561
|
-
"Mme",
|
|
8562
|
-
"Monsieur",
|
|
8563
|
-
"Mr",
|
|
8564
|
-
"Mrs",
|
|
8565
|
-
"Ms",
|
|
8566
|
-
"Pr",
|
|
8567
|
-
"Pr.",
|
|
8568
|
-
"President",
|
|
8569
|
-
"Señor",
|
|
8570
|
-
"Señora",
|
|
8571
|
-
"Sig.",
|
|
8572
|
-
"Sig.ra",
|
|
8573
|
-
"Signor",
|
|
8574
|
-
"Signora",
|
|
8575
|
-
"Signorina",
|
|
8576
|
-
"Sir",
|
|
8577
|
-
"Sr.",
|
|
8578
|
-
"Sra."
|
|
8579
|
-
];
|
|
8580
|
-
/**
|
|
8581
|
-
* Honorifics that need \b word-boundary anchors
|
|
8582
|
-
* (short or common words that could match mid-word).
|
|
8583
|
-
*/
|
|
8584
|
-
const HONORIFIC_BOUNDARY = new Set([
|
|
8585
|
-
"Don",
|
|
8586
|
-
"Doña",
|
|
8587
|
-
"M.",
|
|
8588
|
-
"Me",
|
|
8589
|
-
"Pr",
|
|
8590
|
-
"Pr.",
|
|
8591
|
-
"Señor",
|
|
8592
|
-
"Señora"
|
|
8593
|
-
]);
|
|
8594
|
-
/**
|
|
8595
|
-
* Honorifics that are abbreviations: a dot after them is an
|
|
8596
|
-
* abbreviation dot (same sentence), so the detector keeps an
|
|
8597
|
-
* optional `.` between the title and the name ("Mr. Smith").
|
|
8598
|
-
* Every other honorific is a full word — a trailing dot ends a
|
|
8599
|
-
* sentence, so the detector must NOT consume it (otherwise a span
|
|
8600
|
-
* like "President. The Employee" crosses the sentence boundary).
|
|
8601
|
-
*
|
|
8602
|
-
* Maintained explicitly, NOT derived from "ends in a dot": "Mr" is
|
|
8603
|
-
* an abbreviation written without a dot, and "Lord" is a full word.
|
|
8604
|
-
*/
|
|
8605
|
-
const HONORIFIC_ABBREVIATION = new Set([
|
|
8606
|
-
"Avv.",
|
|
8607
|
-
"Dott.",
|
|
8608
|
-
"M.",
|
|
8609
|
-
"Me",
|
|
8610
|
-
"Messrs",
|
|
8611
|
-
"Mlle",
|
|
8612
|
-
"Mme",
|
|
8613
|
-
"Mr",
|
|
8614
|
-
"Mrs",
|
|
8615
|
-
"Ms",
|
|
8616
|
-
"Pr",
|
|
8617
|
-
"Pr.",
|
|
8618
|
-
"Sig.",
|
|
8619
|
-
"Sig.ra",
|
|
8620
|
-
"Sr.",
|
|
8621
|
-
"Sra."
|
|
8622
|
-
]);
|
|
8623
|
-
/**
|
|
8624
|
-
* Post-nominal degrees (comma or space separated after name).
|
|
8625
|
-
* Plain text; the detector auto-escapes for regex.
|
|
8626
|
-
*/
|
|
8627
|
-
const POST_NOMINALS = [
|
|
8628
|
-
"Ph.D.",
|
|
8629
|
-
"Ph.D",
|
|
8630
|
-
"CSc.",
|
|
8631
|
-
"DrSc.",
|
|
8632
|
-
"ArtD.",
|
|
8633
|
-
"D.Phil.",
|
|
8634
|
-
"DPhil.",
|
|
8635
|
-
"MPhil.",
|
|
8636
|
-
"MBA",
|
|
8637
|
-
"MPA",
|
|
8638
|
-
"LL.M.",
|
|
8639
|
-
"LL.B.",
|
|
8640
|
-
"M.Sc.",
|
|
8641
|
-
"B.Sc.",
|
|
8642
|
-
"MSc.",
|
|
8643
|
-
"BSc.",
|
|
8644
|
-
"M.Eng.",
|
|
8645
|
-
"B.Eng.",
|
|
8646
|
-
"M.A.",
|
|
8647
|
-
"B.A.",
|
|
8648
|
-
"JCD",
|
|
8649
|
-
"JD",
|
|
8650
|
-
"DiS.",
|
|
8651
|
-
"ACCA",
|
|
8652
|
-
"FCCA",
|
|
8653
|
-
"CIPM",
|
|
8654
|
-
"CIPT",
|
|
8655
|
-
"CIPP/E",
|
|
8656
|
-
"CIPP",
|
|
8657
|
-
"KC",
|
|
8658
|
-
"QC"
|
|
8659
|
-
];
|
|
8660
|
-
//#endregion
|
|
8661
9269
|
//#region src/data/amount-words.json
|
|
8662
9270
|
var amount_words_exports = /* @__PURE__ */ __exportAll({
|
|
8663
9271
|
_comment: () => _comment$1,
|
|
@@ -10138,6 +10746,8 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
|
|
|
10138
10746
|
* etc.), skip the comma and degree, then continue.
|
|
10139
10747
|
*/
|
|
10140
10748
|
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
10749
|
+
const PERSON_NAME_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'’\\-]*";
|
|
10750
|
+
const PERSON_NAME_RUN_RE = new RegExp(`^${PERSON_NAME_TOKEN}(?:,?[ \\t]+(?:${`(?:${NAME_PARTICLE}[ \\t]+|d['’])`}){0,2}${PERSON_NAME_TOKEN})*`, "u");
|
|
10141
10751
|
let cachedLegalFormCheckSource = null;
|
|
10142
10752
|
let cachedLegalFormCheckForms = [];
|
|
10143
10753
|
const LEGAL_FORM_ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
@@ -10530,6 +11140,24 @@ const TRIGGER_LOOKAHEAD_MARGIN = 128;
|
|
|
10530
11140
|
const LINE_TRIGGER_LOOKAHEAD = 2048;
|
|
10531
11141
|
const MATCH_PATTERN_LOOKAHEAD = 512;
|
|
10532
11142
|
const PHONE_VALUE_START_RE = /^[+(\d]/;
|
|
11143
|
+
/**
|
|
11144
|
+
* Phone-shaped run: an opening "+", "(", or digit, then
|
|
11145
|
+
* any mix of digits, whitespace, parens, dots, slashes,
|
|
11146
|
+
* and dash variants. The dash class includes every
|
|
11147
|
+
* typographic dash (U+2011 non-breaking hyphen, U+2013
|
|
11148
|
+
* en-dash, etc.) via `DASH_INNER` so a Unicode separator
|
|
11149
|
+
* ("+1 555‑123‑4567") does not break the run and leave the
|
|
11150
|
+
* number unredacted.
|
|
11151
|
+
*/
|
|
11152
|
+
const PHONE_SHAPE_PREFIX_RE = new RegExp(`^[+(\\d][\\d\\s()./${DASH_INNER}]*`);
|
|
11153
|
+
/**
|
|
11154
|
+
* Optional phone-extension suffix following the numeric
|
|
11155
|
+
* run: "ext", "ext.", "extension", or "x" (case-
|
|
11156
|
+
* insensitive) plus 1-6 digits, with optional leading
|
|
11157
|
+
* whitespace. Matches "+1 555 123 4567 ext. 89" and
|
|
11158
|
+
* "555-1234 x42" so the shape bound keeps the extension.
|
|
11159
|
+
*/
|
|
11160
|
+
const PHONE_EXTENSION_SUFFIX_RE = /^\s*(?:ext\.?|extension|x)\s*\d{1,6}/i;
|
|
10533
11161
|
const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}\b/;
|
|
10534
11162
|
const INLINE_FIELD_LABEL_RE = /\b[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
10535
11163
|
const INLINE_FIELD_LABEL_STOP_RE = /(?:^|[^\S\n\t])[\p{L}][\p{L}\p{M} /-]{1,32}:/u;
|
|
@@ -10622,6 +11250,19 @@ const extractValue = (text, triggerEnd, strategy, label) => {
|
|
|
10622
11250
|
end = inlineLabel.index;
|
|
10623
11251
|
foundLineStop = true;
|
|
10624
11252
|
}
|
|
11253
|
+
const region = valueText.slice(0, end);
|
|
11254
|
+
const sentenceBreak = /\.\s/.exec(region);
|
|
11255
|
+
const shapeRegion = sentenceBreak !== null ? region.slice(0, sentenceBreak.index) : region;
|
|
11256
|
+
const shape = PHONE_SHAPE_PREFIX_RE.exec(shapeRegion);
|
|
11257
|
+
if (shape) {
|
|
11258
|
+
let shapeEnd = shape[0].replace(/\D+$/, "").length;
|
|
11259
|
+
const extension = PHONE_EXTENSION_SUFFIX_RE.exec(region.slice(shapeEnd));
|
|
11260
|
+
if (extension !== null) shapeEnd += extension[0].length;
|
|
11261
|
+
if (shapeEnd > 0 && shapeEnd < end) {
|
|
11262
|
+
end = shapeEnd > MAX_TRIGGER_VALUE_LEN ? Math.min(capAtWordBoundary(valueText, MAX_TRIGGER_VALUE_LEN), MAX_TRIGGER_VALUE_LEN) : shapeEnd;
|
|
11263
|
+
foundLineStop = true;
|
|
11264
|
+
}
|
|
11265
|
+
}
|
|
10625
11266
|
}
|
|
10626
11267
|
if (!foundLineStop) end = capAtWordBoundary(valueText, Math.min(end, MAX_TRIGGER_VALUE_LEN));
|
|
10627
11268
|
const rawSlice = valueText.slice(0, end);
|
|
@@ -10800,14 +11441,31 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
10800
11441
|
if (LETTER_RE.test(triggerLastChar) && LETTER_RE.test(fullText[match.end] ?? "")) continue;
|
|
10801
11442
|
const triggerEnd = match.end;
|
|
10802
11443
|
const rawValue = extractValue(fullText, triggerEnd, rule.strategy, rule.label);
|
|
10803
|
-
|
|
11444
|
+
let value = rawValue ? stripQuotes(rawValue) : null;
|
|
10804
11445
|
if (value) {
|
|
10805
11446
|
if (!applyValidations(value.text, rule.validations)) continue;
|
|
10806
11447
|
if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
|
|
11448
|
+
if (rule.label === "phone number" && value.text.length > MAX_TRIGGER_VALUE_LEN && fullText[value.end] !== "\n" && fullText[value.end] !== " ") {
|
|
11449
|
+
const cappedEnd = Math.min(capAtWordBoundary(value.text, MAX_TRIGGER_VALUE_LEN), MAX_TRIGGER_VALUE_LEN);
|
|
11450
|
+
const cappedText = value.text.slice(0, cappedEnd).trimEnd();
|
|
11451
|
+
if (cappedText.length === 0) continue;
|
|
11452
|
+
value = {
|
|
11453
|
+
...value,
|
|
11454
|
+
end: value.start + cappedText.length,
|
|
11455
|
+
text: cappedText
|
|
11456
|
+
};
|
|
11457
|
+
}
|
|
10807
11458
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
10808
|
-
|
|
10809
|
-
|
|
11459
|
+
let entityEnd = value.end;
|
|
11460
|
+
let entityText = fullText.slice(entityStart, entityEnd);
|
|
10810
11461
|
const effectiveLabel = rule.label === "person" && hasKnownLegalFormSuffix(entityText) ? "organization" : rule.label;
|
|
11462
|
+
if (effectiveLabel === "person") {
|
|
11463
|
+
const run = PERSON_NAME_RUN_RE.exec(value.text);
|
|
11464
|
+
if (run !== null && run[0].length < value.text.length) {
|
|
11465
|
+
entityEnd = value.start + run[0].length;
|
|
11466
|
+
entityText = fullText.slice(entityStart, entityEnd);
|
|
11467
|
+
}
|
|
11468
|
+
}
|
|
10811
11469
|
results.push({
|
|
10812
11470
|
start: entityStart,
|
|
10813
11471
|
end: entityEnd,
|
|
@@ -11431,6 +12089,9 @@ const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
|
11431
12089
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
11432
12090
|
let commonWordsPromise = null;
|
|
11433
12091
|
let commonWordsCache = null;
|
|
12092
|
+
const EMPTY_COMMON_WORDS = /* @__PURE__ */ new Set();
|
|
12093
|
+
/** Sync accessor — returns empty set before init. */
|
|
12094
|
+
const getCommonWords = () => commonWordsCache ?? EMPTY_COMMON_WORDS;
|
|
11434
12095
|
const loadCommonWords = () => {
|
|
11435
12096
|
if (commonWordsCache) return Promise.resolve(commonWordsCache);
|
|
11436
12097
|
if (commonWordsPromise) return commonWordsPromise;
|
|
@@ -11691,6 +12352,7 @@ const patternSources = (sources) => {
|
|
|
11691
12352
|
if (sources === void 0) return EMPTY_PATTERN_SOURCES;
|
|
11692
12353
|
return Array.isArray(sources) ? sources : [sources];
|
|
11693
12354
|
};
|
|
12355
|
+
const hasPersonNameSource = (match) => match.sources.includes("first-name") || match.sources.includes("surname");
|
|
11694
12356
|
const addPatternLabel = (list, index, label) => {
|
|
11695
12357
|
const existing = list[index];
|
|
11696
12358
|
if (existing === void 0) {
|
|
@@ -11878,8 +12540,21 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
11878
12540
|
sourceList.push(source);
|
|
11879
12541
|
}
|
|
11880
12542
|
};
|
|
11881
|
-
|
|
11882
|
-
|
|
12543
|
+
const commonWords = getCommonWords();
|
|
12544
|
+
const addDeclinedVariants = (name, source) => {
|
|
12545
|
+
for (const variant of expandNameDeclensions(name)) {
|
|
12546
|
+
if (commonWords.has(variant.toLowerCase())) continue;
|
|
12547
|
+
addNameEntry(variant, source);
|
|
12548
|
+
}
|
|
12549
|
+
};
|
|
12550
|
+
for (const name of getNameCorpusFirstNames(ctx)) {
|
|
12551
|
+
addNameEntry(name, "first-name");
|
|
12552
|
+
addDeclinedVariants(name, "first-name");
|
|
12553
|
+
}
|
|
12554
|
+
for (const name of getNameCorpusSurnames(ctx)) {
|
|
12555
|
+
addNameEntry(name, "surname");
|
|
12556
|
+
addDeclinedVariants(name, "surname");
|
|
12557
|
+
}
|
|
11883
12558
|
for (const title of getNameCorpusTitles(ctx)) {
|
|
11884
12559
|
const norm = stripCuratedPatternSyntax(normalizeForSearch(title));
|
|
11885
12560
|
if (norm.length === 0) continue;
|
|
@@ -12024,6 +12699,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
12024
12699
|
const first = chain.at(0);
|
|
12025
12700
|
const last = chain.at(-1);
|
|
12026
12701
|
if (!first || !last) continue;
|
|
12702
|
+
if (!chain.some(hasPersonNameSource)) continue;
|
|
12027
12703
|
if (isSuppressibleDefinedTermQuote(fullText, first.start, ctx)) continue;
|
|
12028
12704
|
const extended = extendPersonName(fullText, first.start, last.end, ctx);
|
|
12029
12705
|
const score = chain.length >= 2 ? .9 : .5;
|
|
@@ -12656,25 +13332,33 @@ const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-l
|
|
|
12656
13332
|
* to get the base name, and re-scan the full text for
|
|
12657
13333
|
* bare mentions of that base name. Returns new entities
|
|
12658
13334
|
* for occurrences not already covered.
|
|
13335
|
+
*
|
|
13336
|
+
* Propagated mentions are coref aliases: each carries
|
|
13337
|
+
* `corefSourceText` linking it to the full seed entity
|
|
13338
|
+
* text, so placeholder numbering assigns the bare
|
|
13339
|
+
* mention the same placeholder as its source ("Acme"
|
|
13340
|
+
* and "Acme Corp." both become [ORGANIZATION_1]).
|
|
12659
13341
|
*/
|
|
12660
13342
|
const propagateOrgNames = (entities, fullText) => {
|
|
12661
|
-
const
|
|
12662
|
-
const seenBases = /* @__PURE__ */ new Set();
|
|
13343
|
+
const seedByBase = /* @__PURE__ */ new Map();
|
|
12663
13344
|
for (const e of entities) {
|
|
12664
13345
|
if (e.label !== "organization") continue;
|
|
12665
13346
|
if (isCallerOwnedEntity$2(e)) continue;
|
|
12666
13347
|
for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
|
|
12667
13348
|
const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
|
|
12668
|
-
if (base.length >= 3
|
|
12669
|
-
|
|
12670
|
-
|
|
13349
|
+
if (base.length >= 3) {
|
|
13350
|
+
const existing = seedByBase.get(base);
|
|
13351
|
+
if (existing === void 0) seedByBase.set(base, {
|
|
12671
13352
|
baseName: base,
|
|
12672
|
-
label: e.label
|
|
13353
|
+
label: e.label,
|
|
13354
|
+
sourceText: e.text
|
|
12673
13355
|
});
|
|
13356
|
+
else if (existing.sourceText !== e.text) existing.sourceText = base;
|
|
12674
13357
|
}
|
|
12675
13358
|
break;
|
|
12676
13359
|
}
|
|
12677
13360
|
}
|
|
13361
|
+
const seeds = [...seedByBase.values()];
|
|
12678
13362
|
if (seeds.length === 0) return [];
|
|
12679
13363
|
const covered = entities.map((e) => [e.start, e.end]);
|
|
12680
13364
|
const isOverlapping = (start, end) => covered.some(([cs, ce]) => start < ce && end > cs);
|
|
@@ -12708,7 +13392,8 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
12708
13392
|
label,
|
|
12709
13393
|
text: fullText.slice(spanStart, matchEnd),
|
|
12710
13394
|
score: ORG_PROPAGATION_SCORE,
|
|
12711
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
13395
|
+
source: DETECTION_SOURCES.COREFERENCE,
|
|
13396
|
+
corefSourceText: seed.sourceText
|
|
12712
13397
|
});
|
|
12713
13398
|
covered.push([spanStart, matchEnd]);
|
|
12714
13399
|
}
|
|
@@ -13597,6 +14282,7 @@ const applyHotwordRules = (entities, fullText) => {
|
|
|
13597
14282
|
/** Max gap (in chars) between entities to merge. */
|
|
13598
14283
|
const MAX_GAP = 3;
|
|
13599
14284
|
const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
14285
|
+
const hasDetectorLockedBoundary = (entity) => entity.label === "phone number" && entity.source === DETECTION_SOURCES.TRIGGER;
|
|
13600
14286
|
/**
|
|
13601
14287
|
* Characters allowed in the gap between two adjacent
|
|
13602
14288
|
* same-label entities that should be merged: spaces,
|
|
@@ -13777,7 +14463,7 @@ const fixPartialWords = (entities, fullText) => {
|
|
|
13777
14463
|
})).sort((a, b) => a.entity.end - b.entity.end);
|
|
13778
14464
|
const endPositions = byEnd.map((x) => x.entity.end);
|
|
13779
14465
|
return sorted.map((e, eIdx) => {
|
|
13780
|
-
if (hasLockedBoundary$1(e)) return e;
|
|
14466
|
+
if (hasLockedBoundary$1(e) || hasDetectorLockedBoundary(e)) return e;
|
|
13781
14467
|
if (e.text !== fullText.slice(e.start, e.end)) return e;
|
|
13782
14468
|
let newStart = wordStartAt(e.start, boundaries, fullText);
|
|
13783
14469
|
let newEnd = wordEndAt(e.end, boundaries, fullText);
|
|
@@ -14180,6 +14866,8 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
14180
14866
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
14181
14867
|
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
14182
14868
|
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
14869
|
+
const isCoveredByDenyListEntity = (entity, denyListEntity) => entity.label === denyListEntity.label && denyListEntity.start <= entity.start && denyListEntity.end >= entity.end;
|
|
14870
|
+
const filterNameCorpusCoveredByDenyList = (nameCorpusEntities, denyListEntities) => nameCorpusEntities.filter((entity) => !denyListEntities.some((denyListEntity) => isCoveredByDenyListEntity(entity, denyListEntity)));
|
|
14183
14871
|
const LITERAL_BOUNDARY_PUNCT_RE = /^["“„‟‘‛'«]|["”’'»!.]$/u;
|
|
14184
14872
|
const BARE_POSTAL_CODE_RE = /^\s*(?:\d{3}\s?\d{2}|\d{2}[-–]\d{3}|\d{5}(?:[-–]\d{3,4})?)\s*$/u;
|
|
14185
14873
|
const hasCuratedLiteralBoundary = (entity) => LITERAL_SOURCES.has(entity.source) && entity.label !== "person" && entity.sourceDetail !== "gazetteer-extension" && LITERAL_BOUNDARY_PUNCT_RE.test(entity.text);
|
|
@@ -14700,6 +15388,13 @@ const runPipeline = async (options) => {
|
|
|
14700
15388
|
const rawDenyListEntities = config.enableDenyList && search.denyListData ? processDenyListMatches(literalMatches, slices.denyList.start, slices.denyList.end, fullText, search.denyListData, ctx) : [];
|
|
14701
15389
|
const denyListEntities = filterAllowedLabels(rawDenyListEntities, preHotwordAllowedLabels);
|
|
14702
15390
|
if (denyListEntities.length > 0) log("deny-list", `${denyListEntities.length} matches`);
|
|
15391
|
+
if (config.enableNameCorpus && config.enableDenyList) {
|
|
15392
|
+
await initNameCorpus(ctx, config.dictionaries, config.nameCorpusLanguages);
|
|
15393
|
+
checkAbort(signal);
|
|
15394
|
+
rawNameCorpusEntities = filterNameCorpusCoveredByDenyList(detectNameCorpus(fullText, ctx, { mode: "supplemental" }), rawDenyListEntities);
|
|
15395
|
+
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|
|
15396
|
+
log("name-corpus", `${nameCorpusEntities.length} supplemental matches`);
|
|
15397
|
+
}
|
|
14703
15398
|
const rawGazetteerEntities = config.enableGazetteer && search.gazetteerData ? processGazetteerMatches(literalMatches, slices.gazetteer.start, slices.gazetteer.end, fullText, search.gazetteerData) : [];
|
|
14704
15399
|
const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
|
|
14705
15400
|
if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
|
|
@@ -14797,7 +15492,6 @@ const runPipeline = async (options) => {
|
|
|
14797
15492
|
const merged = filterFalsePositives(postOrgEntities, ctx, fullText);
|
|
14798
15493
|
if (merged.length < postOrgEntities.length) log("filter", `removed ${postOrgEntities.length - merged.length} FPs`);
|
|
14799
15494
|
checkAbort(signal);
|
|
14800
|
-
ctx.corefSourceMap.clear();
|
|
14801
15495
|
if (config.enableCoreference) {
|
|
14802
15496
|
if (!legalFormsEnabled) await warmLegalRoleHeads();
|
|
14803
15497
|
const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
|
|
@@ -14863,12 +15557,11 @@ const normalizeEntityText = (label, text) => {
|
|
|
14863
15557
|
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
14864
15558
|
* and N is a 1-based counter per label.
|
|
14865
15559
|
*
|
|
14866
|
-
* @param
|
|
14867
|
-
*
|
|
14868
|
-
*
|
|
14869
|
-
* Defaults to `defaultContext` for single-tenant usage.
|
|
15560
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
15561
|
+
* coref alias links now travel on the entities
|
|
15562
|
+
* themselves (`corefSourceText`).
|
|
14870
15563
|
*/
|
|
14871
|
-
const buildPlaceholderMap = (entities,
|
|
15564
|
+
const buildPlaceholderMap = (entities, _ctx = defaultContext) => {
|
|
14872
15565
|
const counters = /* @__PURE__ */ new Map();
|
|
14873
15566
|
const textLabelToPlaceholder = /* @__PURE__ */ new Map();
|
|
14874
15567
|
const normalizedToPlaceholder = /* @__PURE__ */ new Map();
|
|
@@ -14877,9 +15570,9 @@ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
|
14877
15570
|
const compositeKey = `${entity.label}\0${entity.text}`;
|
|
14878
15571
|
if (textLabelToPlaceholder.has(compositeKey)) continue;
|
|
14879
15572
|
const labelKey = entity.label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
14880
|
-
const sourceText =
|
|
14881
|
-
|
|
14882
|
-
|
|
15573
|
+
const sourceText = entity.source === "coreference" ? entity.corefSourceText : void 0;
|
|
15574
|
+
const sourceNormalizedKey = sourceText === void 0 ? void 0 : `${labelKey}\0${normalizeEntityText(entity.label, sourceText)}`;
|
|
15575
|
+
if (sourceNormalizedKey !== void 0) {
|
|
14883
15576
|
const sourceExisting = normalizedToPlaceholder.get(sourceNormalizedKey);
|
|
14884
15577
|
if (sourceExisting) {
|
|
14885
15578
|
textLabelToPlaceholder.set(compositeKey, sourceExisting);
|
|
@@ -14890,6 +15583,7 @@ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
|
14890
15583
|
const existing = normalizedToPlaceholder.get(normalizedKey);
|
|
14891
15584
|
if (existing) {
|
|
14892
15585
|
textLabelToPlaceholder.set(compositeKey, existing);
|
|
15586
|
+
if (sourceNormalizedKey !== void 0) normalizedToPlaceholder.set(sourceNormalizedKey, existing);
|
|
14893
15587
|
continue;
|
|
14894
15588
|
}
|
|
14895
15589
|
const count = (counters.get(labelKey) ?? 0) + 1;
|
|
@@ -14897,6 +15591,7 @@ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
|
14897
15591
|
const placeholder = `[${labelKey}_${count}]`;
|
|
14898
15592
|
textLabelToPlaceholder.set(compositeKey, placeholder);
|
|
14899
15593
|
normalizedToPlaceholder.set(normalizedKey, placeholder);
|
|
15594
|
+
if (sourceNormalizedKey !== void 0) normalizedToPlaceholder.set(sourceNormalizedKey, placeholder);
|
|
14900
15595
|
}
|
|
14901
15596
|
return textLabelToPlaceholder;
|
|
14902
15597
|
};
|
|
@@ -14939,7 +15634,7 @@ const redactText = (fullText, entities, config = DEFAULT_OPERATOR_CONFIG, ctx =
|
|
|
14939
15634
|
const replacement = operator.apply(entity.text, entity.label, placeholder, config.redactString);
|
|
14940
15635
|
parts.push(replacement);
|
|
14941
15636
|
operatorMap.set(placeholder, opType);
|
|
14942
|
-
if (operator.reversibility === "reversible" && !redactionMap.has(placeholder)) redactionMap.set(placeholder, entity.text);
|
|
15637
|
+
if (operator.reversibility === "reversible" && !redactionMap.has(placeholder)) redactionMap.set(placeholder, entity.source === "coreference" ? entity.corefSourceText : entity.text);
|
|
14943
15638
|
cursor = entity.end;
|
|
14944
15639
|
}
|
|
14945
15640
|
if (cursor < fullText.length) parts.push(fullText.slice(cursor));
|
|
@@ -15395,6 +16090,6 @@ const levenshtein = (rawA, rawB) => {
|
|
|
15395
16090
|
//#region src/wasm.ts
|
|
15396
16091
|
initTextSearch(TextSearch);
|
|
15397
16092
|
//#endregion
|
|
15398
|
-
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
16093
|
+
export { CURRENCY_PATTERN_META, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_REGISTRY, OPERATOR_TYPES, REGEX_META, REGEX_PATTERNS, REGIONS, ZONE_SCORE_ADJUSTMENTS, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, getNameCorpusNonWesternNames, initAddressComponents, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, preparePipelineSearch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText, warmLegalRoleHeads };
|
|
15399
16094
|
|
|
15400
16095
|
//# sourceMappingURL=wasm.mjs.map
|