@stll/anonymize-wasm 1.0.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/address-boundaries.mjs +85 -0
- package/dist/address-boundaries.mjs.map +1 -0
- package/dist/address-prepositions.mjs +182 -0
- package/dist/address-prepositions.mjs.map +1 -0
- package/dist/address-street-types.mjs +162 -0
- package/dist/address-street-types.mjs.map +1 -0
- package/dist/allow-list.mjs +183 -0
- package/dist/allow-list.mjs.map +1 -0
- package/dist/amount-words.mjs +42 -0
- package/dist/amount-words.mjs.map +1 -0
- package/dist/coreference.cs.mjs +14 -0
- package/dist/coreference.cs.mjs.map +1 -0
- package/dist/coreference.de.mjs +14 -0
- package/dist/coreference.de.mjs.map +1 -0
- package/dist/coreference.en.mjs +14 -0
- package/dist/coreference.en.mjs.map +1 -0
- package/dist/coreference.sk.mjs +10 -0
- package/dist/coreference.sk.mjs.map +1 -0
- package/dist/currencies.mjs +231 -0
- package/dist/currencies.mjs.map +1 -0
- package/dist/date-months.mjs +618 -0
- package/dist/date-months.mjs.map +1 -0
- package/dist/generic-roles.mjs +65 -0
- package/dist/generic-roles.mjs.map +1 -0
- package/dist/hotword-rules.mjs +145 -0
- package/dist/hotword-rules.mjs.map +1 -0
- package/dist/legal-forms.mjs +2112 -0
- package/dist/legal-forms.mjs.map +1 -0
- package/dist/manifest.mjs +56 -0
- package/dist/manifest.mjs.map +1 -0
- package/dist/names-exclusions.mjs +191 -0
- package/dist/names-exclusions.mjs.map +1 -0
- package/dist/names-first.mjs +418 -0
- package/dist/names-first.mjs.map +1 -0
- package/dist/names-surnames.mjs +113 -0
- package/dist/names-surnames.mjs.map +1 -0
- package/dist/names-title-tokens.mjs +40 -0
- package/dist/names-title-tokens.mjs.map +1 -0
- package/dist/person-stopwords.mjs +69 -0
- package/dist/person-stopwords.mjs.map +1 -0
- package/dist/section-headings.mjs +52 -0
- package/dist/section-headings.mjs.map +1 -0
- package/dist/signing-clauses.mjs +78 -0
- package/dist/signing-clauses.mjs.map +1 -0
- package/dist/stopwords.mjs +9915 -0
- package/dist/stopwords.mjs.map +1 -0
- package/dist/triggers.cs.mjs +465 -0
- package/dist/triggers.cs.mjs.map +1 -0
- package/dist/triggers.de.mjs +139 -0
- package/dist/triggers.de.mjs.map +1 -0
- package/dist/triggers.en.mjs +72 -0
- package/dist/triggers.en.mjs.map +1 -0
- package/dist/triggers.es.mjs +57 -0
- package/dist/triggers.es.mjs.map +1 -0
- package/dist/triggers.fr.mjs +76 -0
- package/dist/triggers.fr.mjs.map +1 -0
- package/dist/triggers.global.mjs +34 -0
- package/dist/triggers.global.mjs.map +1 -0
- package/dist/triggers.hu.mjs +41 -0
- package/dist/triggers.hu.mjs.map +1 -0
- package/dist/triggers.it.mjs +52 -0
- package/dist/triggers.it.mjs.map +1 -0
- package/dist/triggers.pl.mjs +84 -0
- package/dist/triggers.pl.mjs.map +1 -0
- package/dist/triggers.ro.mjs +59 -0
- package/dist/triggers.ro.mjs.map +1 -0
- package/dist/triggers.sk.mjs +62 -0
- package/dist/triggers.sk.mjs.map +1 -0
- package/dist/triggers.sv.mjs +58 -0
- package/dist/triggers.sv.mjs.map +1 -0
- package/dist/vite.mjs +4 -2
- package/dist/vite.mjs.map +1 -1
- package/dist/wasm.d.mts +75 -11
- package/dist/wasm.mjs +635 -180
- package/dist/wasm.mjs.map +1 -1
- package/dist/year-words.mjs +62 -0
- package/dist/year-words.mjs.map +1 -0
- package/package.json +1 -5
package/dist/wasm.mjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { TextSearch } from "@stll/text-search-wasm";
|
|
2
2
|
import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
|
|
3
3
|
import { toRegex } from "@stll/stdnum/patterns";
|
|
4
|
-
import charGroupsJson from "@stll/anonymize-data/config/char-groups.json";
|
|
5
4
|
//#region src/search-engine.ts
|
|
6
5
|
let _TextSearch;
|
|
7
6
|
const initTextSearch = (ctor) => {
|
|
@@ -125,7 +124,7 @@ const loadManifest = () => {
|
|
|
125
124
|
if (_manifestPromise) return _manifestPromise;
|
|
126
125
|
_manifestPromise = (async () => {
|
|
127
126
|
try {
|
|
128
|
-
const mod = await import("
|
|
127
|
+
const mod = await import("./manifest.mjs");
|
|
129
128
|
const parsed = mod.default ?? mod;
|
|
130
129
|
if (!parsed || typeof parsed.languages !== "object" || parsed.languages === null || Array.isArray(parsed.languages)) {
|
|
131
130
|
console.warn("[anonymize] lang-loader: manifest has unexpected structure, falling back to hardcoded list");
|
|
@@ -144,23 +143,23 @@ const loadManifest = () => {
|
|
|
144
143
|
};
|
|
145
144
|
const LOADER_REGISTRIES = {
|
|
146
145
|
triggers: {
|
|
147
|
-
cs: () => import("
|
|
148
|
-
de: () => import("
|
|
149
|
-
en: () => import("
|
|
150
|
-
es: () => import("
|
|
151
|
-
fr: () => import("
|
|
152
|
-
hu: () => import("
|
|
153
|
-
it: () => import("
|
|
154
|
-
pl: () => import("
|
|
155
|
-
ro: () => import("
|
|
156
|
-
sk: () => import("
|
|
157
|
-
sv: () => import("
|
|
146
|
+
cs: () => import("./triggers.cs.mjs"),
|
|
147
|
+
de: () => import("./triggers.de.mjs"),
|
|
148
|
+
en: () => import("./triggers.en.mjs"),
|
|
149
|
+
es: () => import("./triggers.es.mjs"),
|
|
150
|
+
fr: () => import("./triggers.fr.mjs"),
|
|
151
|
+
hu: () => import("./triggers.hu.mjs"),
|
|
152
|
+
it: () => import("./triggers.it.mjs"),
|
|
153
|
+
pl: () => import("./triggers.pl.mjs"),
|
|
154
|
+
ro: () => import("./triggers.ro.mjs"),
|
|
155
|
+
sk: () => import("./triggers.sk.mjs"),
|
|
156
|
+
sv: () => import("./triggers.sv.mjs")
|
|
158
157
|
},
|
|
159
158
|
coreference: {
|
|
160
|
-
cs: () => import("
|
|
161
|
-
de: () => import("
|
|
162
|
-
en: () => import("
|
|
163
|
-
sk: () => import("
|
|
159
|
+
cs: () => import("./coreference.cs.mjs"),
|
|
160
|
+
de: () => import("./coreference.de.mjs"),
|
|
161
|
+
en: () => import("./coreference.en.mjs"),
|
|
162
|
+
sk: () => import("./coreference.sk.mjs")
|
|
164
163
|
}
|
|
165
164
|
};
|
|
166
165
|
const FALLBACK_LANGUAGES = {
|
|
@@ -231,7 +230,7 @@ const loadLanguageConfigs = async (configType, mapFn) => {
|
|
|
231
230
|
//#region src/detectors/coreference.ts
|
|
232
231
|
/**
|
|
233
232
|
* Load coreference definition patterns from per-language
|
|
234
|
-
* JSON configs in
|
|
233
|
+
* JSON configs in src/data/. Uses the
|
|
235
234
|
* language manifest for auto-discovery.
|
|
236
235
|
*/
|
|
237
236
|
const loadDefinitionPatterns = async () => {
|
|
@@ -264,7 +263,7 @@ const getRoleStopSet = async (ctx) => {
|
|
|
264
263
|
const promise = (async () => {
|
|
265
264
|
let result;
|
|
266
265
|
try {
|
|
267
|
-
const mod = await import("
|
|
266
|
+
const mod = await import("./generic-roles.mjs");
|
|
268
267
|
const data = mod.default ?? mod;
|
|
269
268
|
result = new Set(data.roles.map((r) => r.toLowerCase()));
|
|
270
269
|
} catch {
|
|
@@ -607,58 +606,31 @@ const getNameCorpusFirstNames = (ctx = defaultContext) => ctx.nameCorpus?.firstN
|
|
|
607
606
|
const getNameCorpusSurnames = (ctx = defaultContext) => ctx.nameCorpus?.surnamesList ?? [];
|
|
608
607
|
const getNameCorpusTitles = (ctx = defaultContext) => ctx.nameCorpus?.titlesList ?? [];
|
|
609
608
|
/**
|
|
610
|
-
*
|
|
611
|
-
*
|
|
612
|
-
*/
|
|
613
|
-
const NAME_LANGUAGES = [
|
|
614
|
-
"cs",
|
|
615
|
-
"sk",
|
|
616
|
-
"de",
|
|
617
|
-
"pl",
|
|
618
|
-
"hu",
|
|
619
|
-
"ro",
|
|
620
|
-
"fr",
|
|
621
|
-
"es",
|
|
622
|
-
"it",
|
|
623
|
-
"en",
|
|
624
|
-
"sv"
|
|
625
|
-
];
|
|
626
|
-
/**
|
|
627
|
-
* Try importing a JSON module; return empty array
|
|
628
|
-
* if not found.
|
|
629
|
-
*/
|
|
630
|
-
const tryImportArray = async (path) => {
|
|
631
|
-
try {
|
|
632
|
-
return (await import(path)).default;
|
|
633
|
-
} catch {
|
|
634
|
-
return [];
|
|
635
|
-
}
|
|
636
|
-
};
|
|
637
|
-
/**
|
|
638
|
-
* Load name corpus data from per-language dictionary
|
|
639
|
-
* files and legacy config files. Merges all sources.
|
|
609
|
+
* Load name corpus data from injected dictionaries
|
|
610
|
+
* and legacy config files. Merges all sources.
|
|
640
611
|
*
|
|
641
612
|
* Safe to call multiple times; only loads once per
|
|
642
613
|
* context. Must be called before detectNameCorpus or
|
|
643
614
|
* the getNameCorpus*() accessors are used.
|
|
615
|
+
*
|
|
616
|
+
* @param dictionaries Optional pre-loaded dictionaries
|
|
617
|
+
* with per-language first names and surnames. When
|
|
618
|
+
* omitted, only legacy config files are used.
|
|
644
619
|
*/
|
|
645
|
-
const initNameCorpus = (ctx = defaultContext) => {
|
|
620
|
+
const initNameCorpus = (ctx = defaultContext, dictionaries) => {
|
|
646
621
|
if (ctx.nameCorpusPromise) return ctx.nameCorpusPromise;
|
|
647
622
|
const promise = (async () => {
|
|
648
623
|
try {
|
|
649
624
|
const [legacyFirstMod, legacySurnameMod, titleMod, exclusionMod] = await Promise.all([
|
|
650
|
-
import("
|
|
651
|
-
import("
|
|
652
|
-
import("
|
|
653
|
-
import("
|
|
625
|
+
import("./names-first.mjs"),
|
|
626
|
+
import("./names-surnames.mjs"),
|
|
627
|
+
import("./names-title-tokens.mjs"),
|
|
628
|
+
import("./names-exclusions.mjs")
|
|
654
629
|
]);
|
|
655
|
-
const firstImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/first/${lang}.json`));
|
|
656
|
-
const surnameImports = NAME_LANGUAGES.map((lang) => tryImportArray(`@stll/anonymize-data/dictionaries/names/surnames/${lang}.json`));
|
|
657
|
-
const [firstResults, surnameResults] = await Promise.all([Promise.all(firstImports), Promise.all(surnameImports)]);
|
|
658
630
|
const firstNames = [...legacyFirstMod.default.names];
|
|
659
|
-
for (const names of
|
|
631
|
+
if (dictionaries?.firstNames) for (const names of Object.values(dictionaries.firstNames)) for (const name of names) firstNames.push(name);
|
|
660
632
|
const surnames = [...legacySurnameMod.default.names];
|
|
661
|
-
for (const names of
|
|
633
|
+
if (dictionaries?.surnames) for (const names of Object.values(dictionaries.surnames)) for (const name of names) surnames.push(name);
|
|
662
634
|
const dedup = (arr) => {
|
|
663
635
|
const seen = /* @__PURE__ */ new Set();
|
|
664
636
|
const result = [];
|
|
@@ -1087,12 +1059,515 @@ const POST_NOMINALS = [
|
|
|
1087
1059
|
"CIPP"
|
|
1088
1060
|
];
|
|
1089
1061
|
//#endregion
|
|
1062
|
+
//#region src/data/char-groups.json
|
|
1063
|
+
var char_groups_default = {
|
|
1064
|
+
_comment: "Unicode character equivalence groups. Used to build regex character classes that match all variants of a character type.",
|
|
1065
|
+
groups: {
|
|
1066
|
+
"dash": {
|
|
1067
|
+
"description": "Dash-like characters (hyphens, en-dash, em-dash, minus)",
|
|
1068
|
+
"chars": [
|
|
1069
|
+
{
|
|
1070
|
+
"char": "-",
|
|
1071
|
+
"name": "hyphen-minus",
|
|
1072
|
+
"code": "U+002D"
|
|
1073
|
+
},
|
|
1074
|
+
{
|
|
1075
|
+
"char": "–",
|
|
1076
|
+
"name": "en-dash",
|
|
1077
|
+
"code": "U+2013"
|
|
1078
|
+
},
|
|
1079
|
+
{
|
|
1080
|
+
"char": "—",
|
|
1081
|
+
"name": "em-dash",
|
|
1082
|
+
"code": "U+2014"
|
|
1083
|
+
},
|
|
1084
|
+
{
|
|
1085
|
+
"char": "‐",
|
|
1086
|
+
"name": "hyphen",
|
|
1087
|
+
"code": "U+2010"
|
|
1088
|
+
},
|
|
1089
|
+
{
|
|
1090
|
+
"char": "‑",
|
|
1091
|
+
"name": "non-breaking hyphen",
|
|
1092
|
+
"code": "U+2011"
|
|
1093
|
+
},
|
|
1094
|
+
{
|
|
1095
|
+
"char": "−",
|
|
1096
|
+
"name": "minus sign",
|
|
1097
|
+
"code": "U+2212"
|
|
1098
|
+
},
|
|
1099
|
+
{
|
|
1100
|
+
"char": "⁃",
|
|
1101
|
+
"name": "hyphen bullet",
|
|
1102
|
+
"code": "U+2043"
|
|
1103
|
+
},
|
|
1104
|
+
{
|
|
1105
|
+
"char": "‒",
|
|
1106
|
+
"name": "figure dash",
|
|
1107
|
+
"code": "U+2012"
|
|
1108
|
+
},
|
|
1109
|
+
{
|
|
1110
|
+
"char": "―",
|
|
1111
|
+
"name": "horizontal bar",
|
|
1112
|
+
"code": "U+2015"
|
|
1113
|
+
},
|
|
1114
|
+
{
|
|
1115
|
+
"char": "⸺",
|
|
1116
|
+
"name": "two-em dash",
|
|
1117
|
+
"code": "U+2E3A"
|
|
1118
|
+
},
|
|
1119
|
+
{
|
|
1120
|
+
"char": "⸻",
|
|
1121
|
+
"name": "three-em dash",
|
|
1122
|
+
"code": "U+2E3B"
|
|
1123
|
+
},
|
|
1124
|
+
{
|
|
1125
|
+
"char": "־",
|
|
1126
|
+
"name": "Hebrew maqaf",
|
|
1127
|
+
"code": "U+05BE"
|
|
1128
|
+
}
|
|
1129
|
+
]
|
|
1130
|
+
},
|
|
1131
|
+
"space": {
|
|
1132
|
+
"description": "Space-like characters",
|
|
1133
|
+
"chars": [
|
|
1134
|
+
{
|
|
1135
|
+
"char": " ",
|
|
1136
|
+
"name": "space",
|
|
1137
|
+
"code": "U+0020"
|
|
1138
|
+
},
|
|
1139
|
+
{
|
|
1140
|
+
"char": "\xA0",
|
|
1141
|
+
"name": "no-break space",
|
|
1142
|
+
"code": "U+00A0"
|
|
1143
|
+
},
|
|
1144
|
+
{
|
|
1145
|
+
"char": " ",
|
|
1146
|
+
"name": "en space",
|
|
1147
|
+
"code": "U+2002"
|
|
1148
|
+
},
|
|
1149
|
+
{
|
|
1150
|
+
"char": " ",
|
|
1151
|
+
"name": "em space",
|
|
1152
|
+
"code": "U+2003"
|
|
1153
|
+
},
|
|
1154
|
+
{
|
|
1155
|
+
"char": " ",
|
|
1156
|
+
"name": "thin space",
|
|
1157
|
+
"code": "U+2009"
|
|
1158
|
+
},
|
|
1159
|
+
{
|
|
1160
|
+
"char": " ",
|
|
1161
|
+
"name": "hair space",
|
|
1162
|
+
"code": "U+200A"
|
|
1163
|
+
},
|
|
1164
|
+
{
|
|
1165
|
+
"char": " ",
|
|
1166
|
+
"name": "narrow no-break space",
|
|
1167
|
+
"code": "U+202F"
|
|
1168
|
+
},
|
|
1169
|
+
{
|
|
1170
|
+
"char": " ",
|
|
1171
|
+
"name": "Ogham space mark",
|
|
1172
|
+
"code": "U+1680"
|
|
1173
|
+
},
|
|
1174
|
+
{
|
|
1175
|
+
"char": " ",
|
|
1176
|
+
"name": "en quad",
|
|
1177
|
+
"code": "U+2000"
|
|
1178
|
+
},
|
|
1179
|
+
{
|
|
1180
|
+
"char": " ",
|
|
1181
|
+
"name": "em quad",
|
|
1182
|
+
"code": "U+2001"
|
|
1183
|
+
},
|
|
1184
|
+
{
|
|
1185
|
+
"char": " ",
|
|
1186
|
+
"name": "three-per-em space",
|
|
1187
|
+
"code": "U+2004"
|
|
1188
|
+
},
|
|
1189
|
+
{
|
|
1190
|
+
"char": " ",
|
|
1191
|
+
"name": "four-per-em space",
|
|
1192
|
+
"code": "U+2005"
|
|
1193
|
+
},
|
|
1194
|
+
{
|
|
1195
|
+
"char": " ",
|
|
1196
|
+
"name": "six-per-em space",
|
|
1197
|
+
"code": "U+2006"
|
|
1198
|
+
},
|
|
1199
|
+
{
|
|
1200
|
+
"char": " ",
|
|
1201
|
+
"name": "figure space",
|
|
1202
|
+
"code": "U+2007"
|
|
1203
|
+
},
|
|
1204
|
+
{
|
|
1205
|
+
"char": " ",
|
|
1206
|
+
"name": "punctuation space",
|
|
1207
|
+
"code": "U+2008"
|
|
1208
|
+
},
|
|
1209
|
+
{
|
|
1210
|
+
"char": " ",
|
|
1211
|
+
"name": "medium mathematical space",
|
|
1212
|
+
"code": "U+205F"
|
|
1213
|
+
},
|
|
1214
|
+
{
|
|
1215
|
+
"char": " ",
|
|
1216
|
+
"name": "ideographic space",
|
|
1217
|
+
"code": "U+3000"
|
|
1218
|
+
},
|
|
1219
|
+
{
|
|
1220
|
+
"char": "",
|
|
1221
|
+
"name": "zero width space",
|
|
1222
|
+
"code": "U+200B"
|
|
1223
|
+
}
|
|
1224
|
+
]
|
|
1225
|
+
},
|
|
1226
|
+
"quote-double": {
|
|
1227
|
+
"description": "Double quotation marks",
|
|
1228
|
+
"chars": [
|
|
1229
|
+
{
|
|
1230
|
+
"char": "\"",
|
|
1231
|
+
"name": "quotation mark",
|
|
1232
|
+
"code": "U+0022"
|
|
1233
|
+
},
|
|
1234
|
+
{
|
|
1235
|
+
"char": "“",
|
|
1236
|
+
"name": "left double quotation mark",
|
|
1237
|
+
"code": "U+201C"
|
|
1238
|
+
},
|
|
1239
|
+
{
|
|
1240
|
+
"char": "”",
|
|
1241
|
+
"name": "right double quotation mark",
|
|
1242
|
+
"code": "U+201D"
|
|
1243
|
+
},
|
|
1244
|
+
{
|
|
1245
|
+
"char": "„",
|
|
1246
|
+
"name": "double low-9 quotation mark",
|
|
1247
|
+
"code": "U+201E"
|
|
1248
|
+
},
|
|
1249
|
+
{
|
|
1250
|
+
"char": "«",
|
|
1251
|
+
"name": "left guillemet",
|
|
1252
|
+
"code": "U+00AB"
|
|
1253
|
+
},
|
|
1254
|
+
{
|
|
1255
|
+
"char": "»",
|
|
1256
|
+
"name": "right guillemet",
|
|
1257
|
+
"code": "U+00BB"
|
|
1258
|
+
}
|
|
1259
|
+
]
|
|
1260
|
+
},
|
|
1261
|
+
"quote-single": {
|
|
1262
|
+
"description": "Single quotation marks and apostrophes",
|
|
1263
|
+
"chars": [
|
|
1264
|
+
{
|
|
1265
|
+
"char": "'",
|
|
1266
|
+
"name": "apostrophe",
|
|
1267
|
+
"code": "U+0027"
|
|
1268
|
+
},
|
|
1269
|
+
{
|
|
1270
|
+
"char": "‘",
|
|
1271
|
+
"name": "left single quotation mark",
|
|
1272
|
+
"code": "U+2018"
|
|
1273
|
+
},
|
|
1274
|
+
{
|
|
1275
|
+
"char": "’",
|
|
1276
|
+
"name": "right single quotation mark",
|
|
1277
|
+
"code": "U+2019"
|
|
1278
|
+
},
|
|
1279
|
+
{
|
|
1280
|
+
"char": "‚",
|
|
1281
|
+
"name": "single low-9 quotation mark",
|
|
1282
|
+
"code": "U+201A"
|
|
1283
|
+
},
|
|
1284
|
+
{
|
|
1285
|
+
"char": "‹",
|
|
1286
|
+
"name": "left single guillemet",
|
|
1287
|
+
"code": "U+2039"
|
|
1288
|
+
},
|
|
1289
|
+
{
|
|
1290
|
+
"char": "›",
|
|
1291
|
+
"name": "right single guillemet",
|
|
1292
|
+
"code": "U+203A"
|
|
1293
|
+
},
|
|
1294
|
+
{
|
|
1295
|
+
"char": "ʼ",
|
|
1296
|
+
"name": "modifier letter apostrophe",
|
|
1297
|
+
"code": "U+02BC"
|
|
1298
|
+
},
|
|
1299
|
+
{
|
|
1300
|
+
"char": "´",
|
|
1301
|
+
"name": "acute accent",
|
|
1302
|
+
"code": "U+00B4"
|
|
1303
|
+
},
|
|
1304
|
+
{
|
|
1305
|
+
"char": "`",
|
|
1306
|
+
"name": "grave accent",
|
|
1307
|
+
"code": "U+0060"
|
|
1308
|
+
},
|
|
1309
|
+
{
|
|
1310
|
+
"char": "ʽ",
|
|
1311
|
+
"name": "modifier letter reversed comma",
|
|
1312
|
+
"code": "U+02BD"
|
|
1313
|
+
},
|
|
1314
|
+
{
|
|
1315
|
+
"char": "ʾ",
|
|
1316
|
+
"name": "modifier letter right half ring",
|
|
1317
|
+
"code": "U+02BE"
|
|
1318
|
+
},
|
|
1319
|
+
{
|
|
1320
|
+
"char": "ʿ",
|
|
1321
|
+
"name": "modifier letter left half ring",
|
|
1322
|
+
"code": "U+02BF"
|
|
1323
|
+
},
|
|
1324
|
+
{
|
|
1325
|
+
"char": "ˈ",
|
|
1326
|
+
"name": "modifier letter vertical line",
|
|
1327
|
+
"code": "U+02C8"
|
|
1328
|
+
}
|
|
1329
|
+
]
|
|
1330
|
+
},
|
|
1331
|
+
"dot": {
|
|
1332
|
+
"description": "Dot-like characters",
|
|
1333
|
+
"chars": [
|
|
1334
|
+
{
|
|
1335
|
+
"char": ".",
|
|
1336
|
+
"name": "full stop",
|
|
1337
|
+
"code": "U+002E"
|
|
1338
|
+
},
|
|
1339
|
+
{
|
|
1340
|
+
"char": "·",
|
|
1341
|
+
"name": "middle dot",
|
|
1342
|
+
"code": "U+00B7"
|
|
1343
|
+
},
|
|
1344
|
+
{
|
|
1345
|
+
"char": "•",
|
|
1346
|
+
"name": "bullet",
|
|
1347
|
+
"code": "U+2022"
|
|
1348
|
+
},
|
|
1349
|
+
{
|
|
1350
|
+
"char": "․",
|
|
1351
|
+
"name": "one dot leader",
|
|
1352
|
+
"code": "U+2024"
|
|
1353
|
+
},
|
|
1354
|
+
{
|
|
1355
|
+
"char": "。",
|
|
1356
|
+
"name": "CJK full stop",
|
|
1357
|
+
"code": "U+3002"
|
|
1358
|
+
},
|
|
1359
|
+
{
|
|
1360
|
+
"char": "׃",
|
|
1361
|
+
"name": "Hebrew sof pasuq",
|
|
1362
|
+
"code": "U+05C3"
|
|
1363
|
+
}
|
|
1364
|
+
]
|
|
1365
|
+
},
|
|
1366
|
+
"slash": {
|
|
1367
|
+
"description": "Slash-like characters",
|
|
1368
|
+
"chars": [
|
|
1369
|
+
{
|
|
1370
|
+
"char": "/",
|
|
1371
|
+
"name": "solidus",
|
|
1372
|
+
"code": "U+002F"
|
|
1373
|
+
},
|
|
1374
|
+
{
|
|
1375
|
+
"char": "⁄",
|
|
1376
|
+
"name": "fraction slash",
|
|
1377
|
+
"code": "U+2044"
|
|
1378
|
+
},
|
|
1379
|
+
{
|
|
1380
|
+
"char": "∕",
|
|
1381
|
+
"name": "division slash",
|
|
1382
|
+
"code": "U+2215"
|
|
1383
|
+
}
|
|
1384
|
+
]
|
|
1385
|
+
},
|
|
1386
|
+
"colon": {
|
|
1387
|
+
"description": "Colon-like characters",
|
|
1388
|
+
"chars": [
|
|
1389
|
+
{
|
|
1390
|
+
"char": ":",
|
|
1391
|
+
"name": "colon",
|
|
1392
|
+
"code": "U+003A"
|
|
1393
|
+
},
|
|
1394
|
+
{
|
|
1395
|
+
"char": "꞉",
|
|
1396
|
+
"name": "modifier letter colon",
|
|
1397
|
+
"code": "U+A789"
|
|
1398
|
+
},
|
|
1399
|
+
{
|
|
1400
|
+
"char": "∶",
|
|
1401
|
+
"name": "ratio",
|
|
1402
|
+
"code": "U+2236"
|
|
1403
|
+
},
|
|
1404
|
+
{
|
|
1405
|
+
"char": ":",
|
|
1406
|
+
"name": "full-width colon",
|
|
1407
|
+
"code": "U+FF1A"
|
|
1408
|
+
},
|
|
1409
|
+
{
|
|
1410
|
+
"char": "፥",
|
|
1411
|
+
"name": "Ethiopic colon",
|
|
1412
|
+
"code": "U+1365"
|
|
1413
|
+
}
|
|
1414
|
+
]
|
|
1415
|
+
},
|
|
1416
|
+
"comma": {
|
|
1417
|
+
"description": "Comma-like characters",
|
|
1418
|
+
"chars": [
|
|
1419
|
+
{
|
|
1420
|
+
"char": ",",
|
|
1421
|
+
"name": "comma",
|
|
1422
|
+
"code": "U+002C"
|
|
1423
|
+
},
|
|
1424
|
+
{
|
|
1425
|
+
"char": "‚",
|
|
1426
|
+
"name": "single low-9 quotation mark (used as comma)",
|
|
1427
|
+
"code": "U+201A"
|
|
1428
|
+
},
|
|
1429
|
+
{
|
|
1430
|
+
"char": "،",
|
|
1431
|
+
"name": "Arabic comma",
|
|
1432
|
+
"code": "U+060C"
|
|
1433
|
+
},
|
|
1434
|
+
{
|
|
1435
|
+
"char": "、",
|
|
1436
|
+
"name": "ideographic comma",
|
|
1437
|
+
"code": "U+3001"
|
|
1438
|
+
},
|
|
1439
|
+
{
|
|
1440
|
+
"char": ",",
|
|
1441
|
+
"name": "full-width comma",
|
|
1442
|
+
"code": "U+FF0C"
|
|
1443
|
+
},
|
|
1444
|
+
{
|
|
1445
|
+
"char": "﹐",
|
|
1446
|
+
"name": "small comma",
|
|
1447
|
+
"code": "U+FE50"
|
|
1448
|
+
},
|
|
1449
|
+
{
|
|
1450
|
+
"char": "՝",
|
|
1451
|
+
"name": "Armenian comma",
|
|
1452
|
+
"code": "U+055D"
|
|
1453
|
+
}
|
|
1454
|
+
]
|
|
1455
|
+
},
|
|
1456
|
+
"ellipsis": {
|
|
1457
|
+
"description": "Ellipsis characters",
|
|
1458
|
+
"chars": [{
|
|
1459
|
+
"char": "…",
|
|
1460
|
+
"name": "horizontal ellipsis",
|
|
1461
|
+
"code": "U+2026"
|
|
1462
|
+
}]
|
|
1463
|
+
},
|
|
1464
|
+
"question-mark": {
|
|
1465
|
+
"description": "Question mark variants",
|
|
1466
|
+
"chars": [
|
|
1467
|
+
{
|
|
1468
|
+
"char": "?",
|
|
1469
|
+
"name": "question mark",
|
|
1470
|
+
"code": "U+003F"
|
|
1471
|
+
},
|
|
1472
|
+
{
|
|
1473
|
+
"char": "?",
|
|
1474
|
+
"name": "full-width question mark",
|
|
1475
|
+
"code": "U+FF1F"
|
|
1476
|
+
},
|
|
1477
|
+
{
|
|
1478
|
+
"char": "⁇",
|
|
1479
|
+
"name": "double question mark",
|
|
1480
|
+
"code": "U+2047"
|
|
1481
|
+
},
|
|
1482
|
+
{
|
|
1483
|
+
"char": "⁈",
|
|
1484
|
+
"name": "question exclamation mark",
|
|
1485
|
+
"code": "U+2048"
|
|
1486
|
+
},
|
|
1487
|
+
{
|
|
1488
|
+
"char": "؟",
|
|
1489
|
+
"name": "Arabic question mark",
|
|
1490
|
+
"code": "U+061F"
|
|
1491
|
+
},
|
|
1492
|
+
{
|
|
1493
|
+
"char": "‽",
|
|
1494
|
+
"name": "interrobang",
|
|
1495
|
+
"code": "U+203D"
|
|
1496
|
+
}
|
|
1497
|
+
]
|
|
1498
|
+
},
|
|
1499
|
+
"exclamation-mark": {
|
|
1500
|
+
"description": "Exclamation mark variants",
|
|
1501
|
+
"chars": [
|
|
1502
|
+
{
|
|
1503
|
+
"char": "!",
|
|
1504
|
+
"name": "exclamation mark",
|
|
1505
|
+
"code": "U+0021"
|
|
1506
|
+
},
|
|
1507
|
+
{
|
|
1508
|
+
"char": "!",
|
|
1509
|
+
"name": "full-width exclamation mark",
|
|
1510
|
+
"code": "U+FF01"
|
|
1511
|
+
},
|
|
1512
|
+
{
|
|
1513
|
+
"char": "‼",
|
|
1514
|
+
"name": "double exclamation mark",
|
|
1515
|
+
"code": "U+203C"
|
|
1516
|
+
},
|
|
1517
|
+
{
|
|
1518
|
+
"char": "¡",
|
|
1519
|
+
"name": "inverted exclamation mark",
|
|
1520
|
+
"code": "U+00A1"
|
|
1521
|
+
}
|
|
1522
|
+
]
|
|
1523
|
+
},
|
|
1524
|
+
"semicolon": {
|
|
1525
|
+
"description": "Semicolon variants",
|
|
1526
|
+
"chars": [
|
|
1527
|
+
{
|
|
1528
|
+
"char": ";",
|
|
1529
|
+
"name": "semicolon",
|
|
1530
|
+
"code": "U+003B"
|
|
1531
|
+
},
|
|
1532
|
+
{
|
|
1533
|
+
"char": "؛",
|
|
1534
|
+
"name": "Arabic semicolon",
|
|
1535
|
+
"code": "U+061B"
|
|
1536
|
+
},
|
|
1537
|
+
{
|
|
1538
|
+
"char": ";",
|
|
1539
|
+
"name": "full-width semicolon",
|
|
1540
|
+
"code": "U+FF1B"
|
|
1541
|
+
},
|
|
1542
|
+
{
|
|
1543
|
+
"char": "፤",
|
|
1544
|
+
"name": "Ethiopic semicolon",
|
|
1545
|
+
"code": "U+1364"
|
|
1546
|
+
}
|
|
1547
|
+
]
|
|
1548
|
+
},
|
|
1549
|
+
"underscore": {
|
|
1550
|
+
"description": "Underscore variants",
|
|
1551
|
+
"chars": [{
|
|
1552
|
+
"char": "_",
|
|
1553
|
+
"name": "low line",
|
|
1554
|
+
"code": "U+005F"
|
|
1555
|
+
}, {
|
|
1556
|
+
"char": "_",
|
|
1557
|
+
"name": "full-width low line",
|
|
1558
|
+
"code": "U+FF3F"
|
|
1559
|
+
}]
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
};
|
|
1563
|
+
//#endregion
|
|
1090
1564
|
//#region src/util/char-groups.ts
|
|
1091
1565
|
/**
|
|
1092
1566
|
* Centralized Unicode character equivalence groups.
|
|
1093
1567
|
*
|
|
1094
|
-
*
|
|
1095
|
-
*
|
|
1568
|
+
* Centralized Unicode character equivalence groups.
|
|
1569
|
+
*
|
|
1570
|
+
* Provides helpers to build regex character classes that
|
|
1096
1571
|
* match all typographic variants of a character type
|
|
1097
1572
|
* (dashes, spaces, quotes, etc.).
|
|
1098
1573
|
*
|
|
@@ -1102,7 +1577,7 @@ const POST_NOMINALS = [
|
|
|
1102
1577
|
/** Chars that need escaping inside a regex char class. */
|
|
1103
1578
|
const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
|
|
1104
1579
|
const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
|
|
1105
|
-
const config =
|
|
1580
|
+
const config = char_groups_default;
|
|
1106
1581
|
/**
|
|
1107
1582
|
* Get the raw characters for a named group.
|
|
1108
1583
|
* Throws if the group does not exist.
|
|
@@ -1441,7 +1916,7 @@ const buildDatePatternsFromMonths = (alt) => {
|
|
|
1441
1916
|
/** Cached promise for date patterns. Loaded once. */
|
|
1442
1917
|
let datePatternPromise = null;
|
|
1443
1918
|
const loadDatePatterns = async () => {
|
|
1444
|
-
const mod = await import("
|
|
1919
|
+
const mod = await import("./date-months.mjs");
|
|
1445
1920
|
return buildDatePatternsFromMonths(buildMonthAlternation(mod.default ?? mod));
|
|
1446
1921
|
};
|
|
1447
1922
|
/**
|
|
@@ -1508,7 +1983,7 @@ const buildCurrencyPatterns = (data) => {
|
|
|
1508
1983
|
/** Cached promise for currency patterns. Loaded once. */
|
|
1509
1984
|
let currencyPatternPromise = null;
|
|
1510
1985
|
const loadCurrencyPatterns = async () => {
|
|
1511
|
-
const mod = await import("
|
|
1986
|
+
const mod = await import("./currencies.mjs");
|
|
1512
1987
|
return buildCurrencyPatterns(mod.default ?? mod);
|
|
1513
1988
|
};
|
|
1514
1989
|
/**
|
|
@@ -1590,7 +2065,7 @@ const SIGNING_CLAUSE_META = {
|
|
|
1590
2065
|
};
|
|
1591
2066
|
let signingPatternPromise = null;
|
|
1592
2067
|
const loadSigningPatterns = async () => {
|
|
1593
|
-
const mod = await import("
|
|
2068
|
+
const mod = await import("./signing-clauses.mjs");
|
|
1594
2069
|
return buildSigningClausePatterns(mod.default ?? mod);
|
|
1595
2070
|
};
|
|
1596
2071
|
const getSigningClausePatterns = () => {
|
|
@@ -1624,7 +2099,7 @@ const buildPatternString = (forms) => {
|
|
|
1624
2099
|
const buildLegalFormPatterns = async () => {
|
|
1625
2100
|
let data = {};
|
|
1626
2101
|
try {
|
|
1627
|
-
data = (await import("
|
|
2102
|
+
data = (await import("./legal-forms.mjs")).default;
|
|
1628
2103
|
} catch {
|
|
1629
2104
|
return [];
|
|
1630
2105
|
}
|
|
@@ -1942,14 +2417,14 @@ const buildTriggerPatterns = async () => {
|
|
|
1942
2417
|
rules.push(...expandTriggerGroups(groups));
|
|
1943
2418
|
}
|
|
1944
2419
|
try {
|
|
1945
|
-
const globalMod = await import("
|
|
2420
|
+
const globalMod = await import("./triggers.global.mjs");
|
|
1946
2421
|
const globalGroups = globalMod.default ?? globalMod;
|
|
1947
2422
|
if (Array.isArray(globalGroups)) rules.push(...expandTriggerGroups(globalGroups));
|
|
1948
2423
|
} catch (err) {
|
|
1949
2424
|
if (!(err instanceof Error) || !err.message.includes("Cannot find module")) throw err;
|
|
1950
2425
|
}
|
|
1951
2426
|
try {
|
|
1952
|
-
const yearMod = await import("
|
|
2427
|
+
const yearMod = await import("./year-words.mjs");
|
|
1953
2428
|
const data = yearMod.default ?? yearMod;
|
|
1954
2429
|
const seen = /* @__PURE__ */ new Set();
|
|
1955
2430
|
const yearValidation = compileValidations([{
|
|
@@ -2566,7 +3041,7 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
2566
3041
|
if (ctx.genericRolesPromise) return ctx.genericRolesPromise;
|
|
2567
3042
|
ctx.genericRolesPromise = (async () => {
|
|
2568
3043
|
try {
|
|
2569
|
-
const mod = await import("
|
|
3044
|
+
const mod = await import("./generic-roles.mjs");
|
|
2570
3045
|
const set = new Set(mod.default?.roles ?? []);
|
|
2571
3046
|
ctx.genericRoles = set;
|
|
2572
3047
|
return set;
|
|
@@ -2654,22 +3129,11 @@ const normalizeForSearch = (text) => {
|
|
|
2654
3129
|
};
|
|
2655
3130
|
//#endregion
|
|
2656
3131
|
//#region src/detectors/deny-list.ts
|
|
2657
|
-
/**
|
|
2658
|
-
* Try to load the optional @stll/anonymize-data package.
|
|
2659
|
-
* Returns null if not installed.
|
|
2660
|
-
*/
|
|
2661
|
-
const loadDataModule = async () => {
|
|
2662
|
-
try {
|
|
2663
|
-
return await import("@stll/anonymize-data");
|
|
2664
|
-
} catch {
|
|
2665
|
-
return null;
|
|
2666
|
-
}
|
|
2667
|
-
};
|
|
2668
3132
|
const loadAllowList = (ctx) => {
|
|
2669
3133
|
if (ctx.allowListPromise) return ctx.allowListPromise;
|
|
2670
3134
|
ctx.allowListPromise = (async () => {
|
|
2671
3135
|
try {
|
|
2672
|
-
const mod = await import("
|
|
3136
|
+
const mod = await import("./allow-list.mjs");
|
|
2673
3137
|
const set = new Set(mod.default?.words ?? []);
|
|
2674
3138
|
ctx.allowList = set;
|
|
2675
3139
|
return set;
|
|
@@ -2744,7 +3208,7 @@ const loadStopwords = (ctx) => {
|
|
|
2744
3208
|
if (ctx.stopwordsPromise) return ctx.stopwordsPromise;
|
|
2745
3209
|
ctx.stopwordsPromise = (async () => {
|
|
2746
3210
|
try {
|
|
2747
|
-
const list = ((await import("
|
|
3211
|
+
const list = ((await import("./stopwords.mjs")).default ?? []).filter((w) => !getFirstNameExclusions(ctx).has(w));
|
|
2748
3212
|
const set = new Set(list);
|
|
2749
3213
|
ctx.stopwords = set;
|
|
2750
3214
|
return set;
|
|
@@ -2764,7 +3228,7 @@ const loadPersonStopwords = (ctx) => {
|
|
|
2764
3228
|
if (ctx.personStopwordsPromise) return ctx.personStopwordsPromise;
|
|
2765
3229
|
ctx.personStopwordsPromise = (async () => {
|
|
2766
3230
|
try {
|
|
2767
|
-
const mod = await import("
|
|
3231
|
+
const mod = await import("./person-stopwords.mjs");
|
|
2768
3232
|
const set = new Set(mod.default?.words ?? []);
|
|
2769
3233
|
ctx.personStopwords = set;
|
|
2770
3234
|
return set;
|
|
@@ -2783,46 +3247,33 @@ const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
|
2783
3247
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
2784
3248
|
/**
|
|
2785
3249
|
* Resolve which dictionaries to load based on country
|
|
2786
|
-
* and category filters,
|
|
2787
|
-
*
|
|
2788
|
-
*
|
|
3250
|
+
* and category filters, then build the deny list data.
|
|
3251
|
+
* The returned data provides PatternEntry[] for the
|
|
3252
|
+
* unified builder and parallel arrays for
|
|
2789
3253
|
* post-processing.
|
|
2790
3254
|
*
|
|
2791
|
-
*
|
|
2792
|
-
* Returns null if
|
|
3255
|
+
* Dictionary data is injected via `config.dictionaries`.
|
|
3256
|
+
* Returns null if no dictionaries are provided.
|
|
2793
3257
|
*/
|
|
2794
3258
|
const buildDenyList = async (config, ctx = defaultContext) => {
|
|
2795
|
-
await initNameCorpus(ctx);
|
|
3259
|
+
await initNameCorpus(ctx, config.dictionaries);
|
|
2796
3260
|
await Promise.all([
|
|
2797
3261
|
loadStopwords(ctx),
|
|
2798
3262
|
loadAllowList(ctx),
|
|
2799
3263
|
loadPersonStopwords(ctx),
|
|
2800
3264
|
loadGenericRoles(ctx)
|
|
2801
3265
|
]);
|
|
2802
|
-
const
|
|
2803
|
-
|
|
3266
|
+
const dictionaries = config.dictionaries;
|
|
3267
|
+
const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
|
|
3268
|
+
const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
|
|
3269
|
+
if (!hasDenyList && !hasCities) return buildNameCorpusOnly(config, ctx);
|
|
2804
3270
|
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
2805
3271
|
const excluded = config.denyListExcludeCategories;
|
|
2806
3272
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
2807
|
-
const ids = [...dataModule.ALL_DICTIONARY_IDS].filter((id) => {
|
|
2808
|
-
const meta = dataModule.DICTIONARY_META[id];
|
|
2809
|
-
if (!meta) return false;
|
|
2810
|
-
if (!config.enableNameCorpus && meta.category === "Names") return false;
|
|
2811
|
-
if (excludeCategories.has(meta.category)) return false;
|
|
2812
|
-
if (allowedCountries === null) return true;
|
|
2813
|
-
if (meta.country === null) return true;
|
|
2814
|
-
return allowedCountries.has(meta.country);
|
|
2815
|
-
});
|
|
2816
3273
|
const patternList = [];
|
|
2817
3274
|
const labelList = [];
|
|
2818
3275
|
const sourceList = [];
|
|
2819
3276
|
const patternIndex = /* @__PURE__ */ new Map();
|
|
2820
|
-
const results = await Promise.all(ids.map(async (id) => {
|
|
2821
|
-
return {
|
|
2822
|
-
id,
|
|
2823
|
-
entries: await dataModule.loadDictionary(id)
|
|
2824
|
-
};
|
|
2825
|
-
}));
|
|
2826
3277
|
const addDenyListEntry = (entry, label) => {
|
|
2827
3278
|
const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
2828
3279
|
if (normalized.length === 0) return;
|
|
@@ -2838,47 +3289,59 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
2838
3289
|
sourceList.push(["deny-list"]);
|
|
2839
3290
|
}
|
|
2840
3291
|
};
|
|
2841
|
-
|
|
2842
|
-
const
|
|
2843
|
-
|
|
2844
|
-
for (const
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
"CH",
|
|
2855
|
-
"CZ",
|
|
2856
|
-
"DE",
|
|
2857
|
-
"DK",
|
|
2858
|
-
"ES",
|
|
2859
|
-
"FI",
|
|
2860
|
-
"FR",
|
|
2861
|
-
"GB",
|
|
2862
|
-
"GR",
|
|
2863
|
-
"HR",
|
|
2864
|
-
"HU",
|
|
2865
|
-
"IE",
|
|
2866
|
-
"IT",
|
|
2867
|
-
"LU",
|
|
2868
|
-
"NL",
|
|
2869
|
-
"NO",
|
|
2870
|
-
"NZ",
|
|
2871
|
-
"PL",
|
|
2872
|
-
"PT",
|
|
2873
|
-
"RO",
|
|
2874
|
-
"SE",
|
|
2875
|
-
"SI",
|
|
2876
|
-
"SK",
|
|
2877
|
-
"US"
|
|
2878
|
-
];
|
|
2879
|
-
const cityEntries = await dataModule.loadCityDictionaries(cityCountries);
|
|
2880
|
-
for (const entry of cityEntries) addDenyListEntry(entry, "address");
|
|
3292
|
+
if (hasDenyList) {
|
|
3293
|
+
const denyListData = dictionaries.denyList;
|
|
3294
|
+
const metaData = dictionaries.denyListMeta;
|
|
3295
|
+
for (const [id, entries] of Object.entries(denyListData)) {
|
|
3296
|
+
const meta = metaData[id];
|
|
3297
|
+
if (!meta) continue;
|
|
3298
|
+
if (!config.enableNameCorpus && meta.category === "Names") continue;
|
|
3299
|
+
if (excludeCategories.has(meta.category)) continue;
|
|
3300
|
+
if (allowedCountries !== null && meta.country !== null) {
|
|
3301
|
+
if (!allowedCountries.has(meta.country)) continue;
|
|
3302
|
+
}
|
|
3303
|
+
for (const entry of entries) addDenyListEntry(entry, meta.label);
|
|
3304
|
+
}
|
|
2881
3305
|
}
|
|
3306
|
+
if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
|
|
3307
|
+
appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, patternIndex);
|
|
3308
|
+
if (patternList.length === 0) return null;
|
|
3309
|
+
return {
|
|
3310
|
+
labels: labelList,
|
|
3311
|
+
originals: patternList,
|
|
3312
|
+
sources: sourceList
|
|
3313
|
+
};
|
|
3314
|
+
};
|
|
3315
|
+
/**
|
|
3316
|
+
* Build deny-list data containing only name corpus
|
|
3317
|
+
* entries (no external dictionaries). Used when no
|
|
3318
|
+
* dictionary data is injected but name corpus is
|
|
3319
|
+
* enabled.
|
|
3320
|
+
*/
|
|
3321
|
+
const buildNameCorpusOnly = (config, ctx) => {
|
|
3322
|
+
if (!config.enableNameCorpus) return null;
|
|
3323
|
+
const excluded = config.denyListExcludeCategories;
|
|
3324
|
+
if ((excluded ? new Set(excluded) : /* @__PURE__ */ new Set()).has("Names")) return null;
|
|
3325
|
+
const patternList = [];
|
|
3326
|
+
const labelList = [];
|
|
3327
|
+
const sourceList = [];
|
|
3328
|
+
appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, /* @__PURE__ */ new Map());
|
|
3329
|
+
if (patternList.length === 0) return null;
|
|
3330
|
+
return {
|
|
3331
|
+
labels: labelList,
|
|
3332
|
+
originals: patternList,
|
|
3333
|
+
sources: sourceList
|
|
3334
|
+
};
|
|
3335
|
+
};
|
|
3336
|
+
/**
|
|
3337
|
+
* Append name corpus entries (first names, surnames,
|
|
3338
|
+
* titles) to the pattern arrays. Shared between
|
|
3339
|
+
* buildDenyList and buildNameCorpusOnly.
|
|
3340
|
+
*/
|
|
3341
|
+
const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList, patternIndex) => {
|
|
3342
|
+
const excluded = config.denyListExcludeCategories;
|
|
3343
|
+
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
3344
|
+
if (!config.enableNameCorpus || excludeCategories.has("Names")) return;
|
|
2882
3345
|
const addNameEntry = (name, source) => {
|
|
2883
3346
|
const normalized = normalizeForSearch(name).replace(/[|\\]/g, "");
|
|
2884
3347
|
if (normalized.length === 0) return;
|
|
@@ -2894,30 +3357,22 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
2894
3357
|
sourceList.push([source]);
|
|
2895
3358
|
}
|
|
2896
3359
|
};
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
if (existing
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
sourceList.push(["title"]);
|
|
2912
|
-
}
|
|
3360
|
+
for (const name of getNameCorpusFirstNames(ctx)) addNameEntry(name, "first-name");
|
|
3361
|
+
for (const name of getNameCorpusSurnames(ctx)) addNameEntry(name, "surname");
|
|
3362
|
+
for (const title of getNameCorpusTitles(ctx)) {
|
|
3363
|
+
const norm = normalizeForSearch(title).replace(/[|\\]/g, "");
|
|
3364
|
+
if (norm.length === 0) continue;
|
|
3365
|
+
const lower = norm.toLowerCase();
|
|
3366
|
+
const existing = patternIndex.get(lower);
|
|
3367
|
+
if (existing !== void 0) {
|
|
3368
|
+
if (!sourceList[existing].includes("title")) sourceList[existing].push("title");
|
|
3369
|
+
} else {
|
|
3370
|
+
patternIndex.set(lower, patternList.length);
|
|
3371
|
+
patternList.push(norm);
|
|
3372
|
+
labelList.push(["person"]);
|
|
3373
|
+
sourceList.push(["title"]);
|
|
2913
3374
|
}
|
|
2914
3375
|
}
|
|
2915
|
-
if (patternList.length === 0) return null;
|
|
2916
|
-
return {
|
|
2917
|
-
labels: labelList,
|
|
2918
|
-
originals: patternList,
|
|
2919
|
-
sources: sourceList
|
|
2920
|
-
};
|
|
2921
3376
|
};
|
|
2922
3377
|
/**
|
|
2923
3378
|
* Ensure all deny-list support data (stopwords, allow
|
|
@@ -2927,8 +3382,8 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
2927
3382
|
* the search instance was built on a different context
|
|
2928
3383
|
* (e.g. cachedSearch).
|
|
2929
3384
|
*/
|
|
2930
|
-
const ensureDenyListData = async (ctx = defaultContext) => {
|
|
2931
|
-
await initNameCorpus(ctx);
|
|
3385
|
+
const ensureDenyListData = async (ctx = defaultContext, dictionaries) => {
|
|
3386
|
+
await initNameCorpus(ctx, dictionaries);
|
|
2932
3387
|
await Promise.all([
|
|
2933
3388
|
loadStopwords(ctx),
|
|
2934
3389
|
loadAllowList(ctx),
|
|
@@ -3137,7 +3592,7 @@ const extendPersonName = (text, start, end, ctx) => {
|
|
|
3137
3592
|
let cachedBoundaryRe = null;
|
|
3138
3593
|
const loadBoundaryWords = async () => {
|
|
3139
3594
|
try {
|
|
3140
|
-
return (await import("
|
|
3595
|
+
return (await import("./address-boundaries.mjs")).default;
|
|
3141
3596
|
} catch {
|
|
3142
3597
|
return {};
|
|
3143
3598
|
}
|
|
@@ -3166,7 +3621,7 @@ const getBoundaryRe = async () => {
|
|
|
3166
3621
|
const buildStreetTypePatterns = async () => {
|
|
3167
3622
|
let config = {};
|
|
3168
3623
|
try {
|
|
3169
|
-
config = (await import("
|
|
3624
|
+
config = (await import("./address-street-types.mjs")).default;
|
|
3170
3625
|
} catch {
|
|
3171
3626
|
return [];
|
|
3172
3627
|
}
|
|
@@ -3494,7 +3949,7 @@ let _temporalPreps = null;
|
|
|
3494
3949
|
let _prepsPromise = null;
|
|
3495
3950
|
const loadPrepositions = async () => {
|
|
3496
3951
|
try {
|
|
3497
|
-
const mod = await import("
|
|
3952
|
+
const mod = await import("./address-prepositions.mjs");
|
|
3498
3953
|
const data = mod.default ?? mod;
|
|
3499
3954
|
const addr = /* @__PURE__ */ new Set();
|
|
3500
3955
|
const temp = /* @__PURE__ */ new Set();
|
|
@@ -3518,7 +3973,7 @@ let _streetAbbrevs = null;
|
|
|
3518
3973
|
let _streetAbbrevsPromise = null;
|
|
3519
3974
|
const loadStreetAbbrevs = async () => {
|
|
3520
3975
|
try {
|
|
3521
|
-
const mod = await import("
|
|
3976
|
+
const mod = await import("./address-street-types.mjs");
|
|
3522
3977
|
const data = mod.default ?? mod;
|
|
3523
3978
|
const abbrevs = /* @__PURE__ */ new Set();
|
|
3524
3979
|
for (const [key, words] of Object.entries(data)) {
|
|
@@ -3687,11 +4142,11 @@ const ZONE_SCORE_ADJUSTMENTS = {
|
|
|
3687
4142
|
table: .05
|
|
3688
4143
|
};
|
|
3689
4144
|
const loadSectionHeadings = async () => {
|
|
3690
|
-
const mod = await import("
|
|
4145
|
+
const mod = await import("./section-headings.mjs");
|
|
3691
4146
|
return (mod.default ?? mod).patterns.map((p) => new RegExp(p.re, p.flags));
|
|
3692
4147
|
};
|
|
3693
4148
|
const loadSigningClauses = async () => {
|
|
3694
|
-
const mod = await import("
|
|
4149
|
+
const mod = await import("./signing-clauses.mjs");
|
|
3695
4150
|
return (mod.default ?? mod).patterns.map((p) => {
|
|
3696
4151
|
const prefix = p.prefix || "";
|
|
3697
4152
|
const suffix = p.suffix || "";
|
|
@@ -3875,7 +4330,7 @@ let search = null;
|
|
|
3875
4330
|
let patternToRule = null;
|
|
3876
4331
|
let initPromise = null;
|
|
3877
4332
|
const loadRules = async () => {
|
|
3878
|
-
const mod = await import("
|
|
4333
|
+
const mod = await import("./hotword-rules.mjs");
|
|
3879
4334
|
const loaded = (mod.default ?? mod).rules;
|
|
3880
4335
|
const patterns = [];
|
|
3881
4336
|
const mapping = [];
|
|
@@ -4575,7 +5030,7 @@ let amountWordsLoaded = false;
|
|
|
4575
5030
|
const getAmountWordsRe = async () => {
|
|
4576
5031
|
if (amountWordsLoaded && amountWordsRe) return amountWordsRe;
|
|
4577
5032
|
try {
|
|
4578
|
-
const alt = (await import("
|
|
5033
|
+
const alt = (await import("./amount-words.mjs")).default.patterns.flatMap((p) => p.keywords).map(escapeRegex).join("|");
|
|
4579
5034
|
amountWordsRe = new RegExp(`^[,;]?[^\\S\\n]*(\\((?:${alt})[:\\s][^)\\n]{1,120}\\))`, "i");
|
|
4580
5035
|
} catch {
|
|
4581
5036
|
amountWordsRe = /^[,;]?[^\S\n]*(\((?:slovy|slovně)[:\s][^)\n]{1,120}\))/i;
|
|
@@ -4684,7 +5139,7 @@ const runPipeline = async (options) => {
|
|
|
4684
5139
|
initStreetAbbrevs(),
|
|
4685
5140
|
hotwordInit
|
|
4686
5141
|
]);
|
|
4687
|
-
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx);
|
|
5142
|
+
if (cachedSearch && config.enableDenyList) await ensureDenyListData(ctx, config.dictionaries);
|
|
4688
5143
|
let zones = [];
|
|
4689
5144
|
if (config.enableZoneClassification && zoneInitOk) {
|
|
4690
5145
|
zones = classifyZones(fullText, ctx);
|
|
@@ -4710,7 +5165,7 @@ const runPipeline = async (options) => {
|
|
|
4710
5165
|
let rawNameCorpusEntities = [];
|
|
4711
5166
|
let nameCorpusEntities = [];
|
|
4712
5167
|
if (config.enableNameCorpus && !config.enableDenyList) {
|
|
4713
|
-
await initNameCorpus(ctx);
|
|
5168
|
+
await initNameCorpus(ctx, config.dictionaries);
|
|
4714
5169
|
checkAbort(signal);
|
|
4715
5170
|
rawNameCorpusEntities = detectNameCorpus(fullText, ctx);
|
|
4716
5171
|
nameCorpusEntities = filterAllowedLabels(rawNameCorpusEntities, preHotwordAllowedLabels);
|