@stll/anonymize-wasm 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/wasm.d.mts +45 -6
- package/dist/wasm.mjs +185 -43
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.mjs
CHANGED
|
@@ -2021,19 +2021,21 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
|
|
|
2021
2021
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
2022
2022
|
const meta = meta_[idx - sliceStart];
|
|
2023
2023
|
if (!meta) continue;
|
|
2024
|
-
if (meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2024
|
+
if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
|
|
2025
2025
|
if (meta.validator) {
|
|
2026
2026
|
const compacted = meta.validator.compact(match.text);
|
|
2027
2027
|
if (!meta.validator.validate(compacted).valid) continue;
|
|
2028
2028
|
}
|
|
2029
|
-
|
|
2029
|
+
const entity = {
|
|
2030
2030
|
start: match.start,
|
|
2031
2031
|
end: match.end,
|
|
2032
2032
|
label: meta.label,
|
|
2033
2033
|
text: match.text,
|
|
2034
2034
|
score: meta.score,
|
|
2035
2035
|
source: DETECTION_SOURCES.REGEX
|
|
2036
|
-
}
|
|
2036
|
+
};
|
|
2037
|
+
if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
|
|
2038
|
+
results.push(entity);
|
|
2037
2039
|
}
|
|
2038
2040
|
return results;
|
|
2039
2041
|
};
|
|
@@ -3055,6 +3057,7 @@ const loadGenericRoles = (ctx = defaultContext) => {
|
|
|
3055
3057
|
};
|
|
3056
3058
|
/** Sync accessor — returns empty set before init. */
|
|
3057
3059
|
const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
|
|
3060
|
+
const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3058
3061
|
/**
|
|
3059
3062
|
* Filter out entities that are likely false positives:
|
|
3060
3063
|
* template placeholders, clause/section numbers,
|
|
@@ -3067,9 +3070,17 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
|
|
|
3067
3070
|
const filtered = [];
|
|
3068
3071
|
const roles = getGenericRoles(ctx);
|
|
3069
3072
|
for (const entity of entities) {
|
|
3073
|
+
if (isCallerOwnedEntity$3(entity)) {
|
|
3074
|
+
filtered.push(entity);
|
|
3075
|
+
continue;
|
|
3076
|
+
}
|
|
3070
3077
|
const normalized = normalizeEntity(entity);
|
|
3071
3078
|
if (!normalized) continue;
|
|
3072
3079
|
const trimmed = normalized.text;
|
|
3080
|
+
if (isCallerOwnedEntity$3(normalized)) {
|
|
3081
|
+
filtered.push(normalized);
|
|
3082
|
+
continue;
|
|
3083
|
+
}
|
|
3073
3084
|
if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
|
|
3074
3085
|
const maxLen = MAX_ENTITY_LENGTH[normalized.label];
|
|
3075
3086
|
if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
|
|
@@ -3244,6 +3255,7 @@ const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
|
|
|
3244
3255
|
/** Sync accessor — returns empty set before init. */
|
|
3245
3256
|
const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
|
|
3246
3257
|
const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
|
|
3258
|
+
const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
|
|
3247
3259
|
const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
|
|
3248
3260
|
/**
|
|
3249
3261
|
* Resolve which dictionaries to load based on country
|
|
@@ -3266,27 +3278,31 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
3266
3278
|
const dictionaries = config.dictionaries;
|
|
3267
3279
|
const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
|
|
3268
3280
|
const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
|
|
3269
|
-
|
|
3281
|
+
const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
|
|
3282
|
+
if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
|
|
3270
3283
|
const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
|
|
3271
3284
|
const excluded = config.denyListExcludeCategories;
|
|
3272
3285
|
const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
|
|
3273
3286
|
const patternList = [];
|
|
3274
3287
|
const labelList = [];
|
|
3288
|
+
const customLabelList = [];
|
|
3275
3289
|
const sourceList = [];
|
|
3276
3290
|
const patternIndex = /* @__PURE__ */ new Map();
|
|
3277
|
-
const addDenyListEntry = (entry, label) => {
|
|
3278
|
-
const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
3291
|
+
const addDenyListEntry = (entry, label, source = "deny-list") => {
|
|
3292
|
+
const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
|
|
3279
3293
|
if (normalized.length === 0) return;
|
|
3280
3294
|
const lower = normalized.toLowerCase();
|
|
3281
3295
|
const existing = patternIndex.get(lower);
|
|
3282
3296
|
if (existing !== void 0) {
|
|
3283
3297
|
if (!labelList[existing].includes(label)) labelList[existing].push(label);
|
|
3284
|
-
if (!sourceList[existing].includes(
|
|
3298
|
+
if (!sourceList[existing].includes(source)) sourceList[existing].push(source);
|
|
3299
|
+
if (source === "custom-deny-list" && !customLabelList[existing].includes(label)) customLabelList[existing].push(label);
|
|
3285
3300
|
} else {
|
|
3286
3301
|
patternIndex.set(lower, patternList.length);
|
|
3287
3302
|
patternList.push(normalized);
|
|
3288
3303
|
labelList.push([label]);
|
|
3289
|
-
|
|
3304
|
+
customLabelList.push(source === "custom-deny-list" ? [label] : []);
|
|
3305
|
+
sourceList.push([source]);
|
|
3290
3306
|
}
|
|
3291
3307
|
};
|
|
3292
3308
|
if (hasDenyList) {
|
|
@@ -3304,10 +3320,15 @@ const buildDenyList = async (config, ctx = defaultContext) => {
|
|
|
3304
3320
|
}
|
|
3305
3321
|
}
|
|
3306
3322
|
if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
|
|
3323
|
+
if (hasCustomDenyList) for (const entry of config.customDenyList) {
|
|
3324
|
+
addDenyListEntry(entry.value, entry.label, "custom-deny-list");
|
|
3325
|
+
for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
|
|
3326
|
+
}
|
|
3307
3327
|
appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, patternIndex);
|
|
3308
3328
|
if (patternList.length === 0) return null;
|
|
3309
3329
|
return {
|
|
3310
3330
|
labels: labelList,
|
|
3331
|
+
customLabels: customLabelList,
|
|
3311
3332
|
originals: patternList,
|
|
3312
3333
|
sources: sourceList
|
|
3313
3334
|
};
|
|
@@ -3324,11 +3345,13 @@ const buildNameCorpusOnly = (config, ctx) => {
|
|
|
3324
3345
|
if ((excluded ? new Set(excluded) : /* @__PURE__ */ new Set()).has("Names")) return null;
|
|
3325
3346
|
const patternList = [];
|
|
3326
3347
|
const labelList = [];
|
|
3348
|
+
const customLabelList = [];
|
|
3327
3349
|
const sourceList = [];
|
|
3328
3350
|
appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, /* @__PURE__ */ new Map());
|
|
3329
3351
|
if (patternList.length === 0) return null;
|
|
3330
3352
|
return {
|
|
3331
3353
|
labels: labelList,
|
|
3354
|
+
customLabels: customLabelList,
|
|
3332
3355
|
originals: patternList,
|
|
3333
3356
|
sources: sourceList
|
|
3334
3357
|
};
|
|
@@ -3374,6 +3397,14 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
3374
3397
|
}
|
|
3375
3398
|
}
|
|
3376
3399
|
};
|
|
3400
|
+
const customMatchHasValidEdges = (fullText, start, end, pattern) => {
|
|
3401
|
+
if (!WORD_CHAR_RE$1.test(pattern)) return true;
|
|
3402
|
+
const prev = fullText[start - 1] ?? "";
|
|
3403
|
+
const next = fullText[end] ?? "";
|
|
3404
|
+
if (WORD_CHAR_RE$1.test(prev)) return false;
|
|
3405
|
+
if (WORD_CHAR_RE$1.test(next)) return false;
|
|
3406
|
+
return true;
|
|
3407
|
+
};
|
|
3377
3408
|
/**
|
|
3378
3409
|
* Ensure all deny-list support data (stopwords, allow
|
|
3379
3410
|
* list, person stopwords, generic roles) is loaded on
|
|
@@ -3411,18 +3442,24 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3411
3442
|
const idx = match.pattern;
|
|
3412
3443
|
if (idx < sliceStart || idx >= sliceEnd) continue;
|
|
3413
3444
|
const localIdx = idx - sliceStart;
|
|
3414
|
-
const
|
|
3415
|
-
if (!UPPER_START_RE.test(sourceChar)) continue;
|
|
3445
|
+
const sources = data.sources[localIdx] ?? [];
|
|
3416
3446
|
const matchText = fullText.slice(match.start, match.end);
|
|
3447
|
+
const sourceChar = fullText[match.start] ?? "";
|
|
3417
3448
|
const keyword = matchText.toLowerCase();
|
|
3418
|
-
if (getStopwords(ctx).has(keyword) || getAllowList(ctx).has(keyword)) continue;
|
|
3419
|
-
if (ALL_UPPER_RE.test(matchText)) continue;
|
|
3420
3449
|
const labels = data.labels[localIdx];
|
|
3421
|
-
|
|
3450
|
+
const pattern = data.originals[localIdx] ?? "";
|
|
3451
|
+
const customPatternLabels = data.customLabels[localIdx] ?? [];
|
|
3452
|
+
const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
|
|
3453
|
+
const customLabels = customEdgesAreValid ? customPatternLabels : [];
|
|
3454
|
+
if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
|
|
3455
|
+
const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
|
|
3456
|
+
if (curatedLabels.length === 0 && customLabels.length === 0) continue;
|
|
3422
3457
|
const entry = {
|
|
3423
3458
|
start: match.start,
|
|
3424
3459
|
end: match.end,
|
|
3425
|
-
labels,
|
|
3460
|
+
labels: curatedLabels,
|
|
3461
|
+
customLabels,
|
|
3462
|
+
sources,
|
|
3426
3463
|
text: matchText,
|
|
3427
3464
|
patternIdx: localIdx
|
|
3428
3465
|
};
|
|
@@ -3433,22 +3470,31 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
|
|
|
3433
3470
|
const results = [];
|
|
3434
3471
|
const nameHits = [];
|
|
3435
3472
|
for (const [, matches] of Array.from(matchesByPattern)) {
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
const hasPerson = first.labels.includes("person");
|
|
3439
|
-
const nonPersonLabels = first.labels.filter((l) => l !== "person");
|
|
3440
|
-
if (hasPerson) {
|
|
3441
|
-
const keyword = first.text.toLowerCase();
|
|
3442
|
-
if (!getPersonStopwords(ctx).has(keyword)) for (const m of matches) nameHits.push(m);
|
|
3443
|
-
}
|
|
3444
|
-
for (const m of matches) for (const label of nonPersonLabels) results.push({
|
|
3473
|
+
if (!matches[0]) continue;
|
|
3474
|
+
for (const m of matches) for (const label of m.customLabels) results.push({
|
|
3445
3475
|
start: m.start,
|
|
3446
3476
|
end: m.end,
|
|
3447
3477
|
label,
|
|
3448
3478
|
text: m.text,
|
|
3449
3479
|
score: .9,
|
|
3450
|
-
source: DETECTION_SOURCES.DENY_LIST
|
|
3480
|
+
source: DETECTION_SOURCES.DENY_LIST,
|
|
3481
|
+
sourceDetail: "custom-deny-list"
|
|
3451
3482
|
});
|
|
3483
|
+
for (const m of matches) {
|
|
3484
|
+
if (m.labels.includes("person")) {
|
|
3485
|
+
const keyword = m.text.toLowerCase();
|
|
3486
|
+
if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
|
|
3487
|
+
}
|
|
3488
|
+
const nonPersonLabels = m.labels.filter((l) => l !== "person");
|
|
3489
|
+
for (const label of nonPersonLabels) results.push({
|
|
3490
|
+
start: m.start,
|
|
3491
|
+
end: m.end,
|
|
3492
|
+
label,
|
|
3493
|
+
text: m.text,
|
|
3494
|
+
score: .9,
|
|
3495
|
+
source: DETECTION_SOURCES.DENY_LIST
|
|
3496
|
+
});
|
|
3497
|
+
}
|
|
3452
3498
|
}
|
|
3453
3499
|
nameHits.sort((a, b) => a.start - b.start);
|
|
3454
3500
|
const nameConsumed = /* @__PURE__ */ new Set();
|
|
@@ -3528,6 +3574,7 @@ const TRAILING_WORD_EXCLUSIONS = new Set([
|
|
|
3528
3574
|
const extendCityDistricts = (entities, fullText) => {
|
|
3529
3575
|
for (const entity of entities) {
|
|
3530
3576
|
if (entity.label !== "address") continue;
|
|
3577
|
+
if (entity.sourceDetail === "custom-deny-list") continue;
|
|
3531
3578
|
const afterMatch = fullText.slice(entity.end);
|
|
3532
3579
|
const suffixM = DISTRICT_SUFFIX_RE.exec(afterMatch);
|
|
3533
3580
|
if (suffixM) {
|
|
@@ -3646,6 +3693,7 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
|
|
|
3646
3693
|
}
|
|
3647
3694
|
for (const e of existingEntities) {
|
|
3648
3695
|
if (e.label !== "address") continue;
|
|
3696
|
+
if (e.sourceDetail === "custom-deny-list") continue;
|
|
3649
3697
|
if (e.source === "deny-list") seeds.push({
|
|
3650
3698
|
type: "city",
|
|
3651
3699
|
start: e.start,
|
|
@@ -3812,6 +3860,7 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
|
|
|
3812
3860
|
const TRAILING_SEP = /[,\s]+$/;
|
|
3813
3861
|
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
|
|
3814
3862
|
const ORG_PROPAGATION_SCORE = .9;
|
|
3863
|
+
const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3815
3864
|
/**
|
|
3816
3865
|
* After the main detection pass, collect organization
|
|
3817
3866
|
* entities with a legal form suffix, strip the suffix
|
|
@@ -3824,6 +3873,7 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
3824
3873
|
const seenBases = /* @__PURE__ */ new Set();
|
|
3825
3874
|
for (const e of entities) {
|
|
3826
3875
|
if (e.label !== "organization") continue;
|
|
3876
|
+
if (isCallerOwnedEntity$2(e)) continue;
|
|
3827
3877
|
for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
|
|
3828
3878
|
const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
|
|
3829
3879
|
if (base.length >= 3 && !seenBases.has(base)) {
|
|
@@ -3903,6 +3953,7 @@ const NEAR_MISS_BAND = .15;
|
|
|
3903
3953
|
const BOOST_PER_NEIGHBOUR = .05;
|
|
3904
3954
|
const CONTEXT_WINDOW_CHARS = 150;
|
|
3905
3955
|
const HIGH_CONFIDENCE_FLOOR = .9;
|
|
3956
|
+
const isCallerOwnedEntity$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
3906
3957
|
/**
|
|
3907
3958
|
* Boost confidence of near-miss NER entities that appear
|
|
3908
3959
|
* near high-confidence detections (regex, trigger phrase).
|
|
@@ -4011,7 +4062,7 @@ const getStreetAbbrevs = () => _streetAbbrevs ?? /* @__PURE__ */ new Set();
|
|
|
4011
4062
|
*/
|
|
4012
4063
|
const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
|
|
4013
4064
|
const results = [];
|
|
4014
|
-
const addressEntities = existingEntities.filter((e) => e.label === "address");
|
|
4065
|
+
const addressEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity))).filter((e) => e.label === "address");
|
|
4015
4066
|
const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
|
|
4016
4067
|
const houseNumRe = /\b(?:\d{1,4}\/\d+[a-zA-Z]\b|\d{3,4}\/\d+\b|(?:1[3-9]|[2-9]\d)\/\d{3,}\b)/g;
|
|
4017
4068
|
houseNumRe.lastIndex = 0;
|
|
@@ -4107,6 +4158,7 @@ const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]
|
|
|
4107
4158
|
*/
|
|
4108
4159
|
const detectOrphanStreetLines = (fullText, existingEntities) => {
|
|
4109
4160
|
const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
|
|
4161
|
+
const contextEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity)));
|
|
4110
4162
|
const results = [];
|
|
4111
4163
|
ORPHAN_STREET_RE.lastIndex = 0;
|
|
4112
4164
|
for (let m = ORPHAN_STREET_RE.exec(fullText); m !== null; m = ORPHAN_STREET_RE.exec(fullText)) {
|
|
@@ -4116,7 +4168,7 @@ const detectOrphanStreetLines = (fullText, existingEntities) => {
|
|
|
4116
4168
|
const end = start + captured.length;
|
|
4117
4169
|
if (start >= headerEnd) continue;
|
|
4118
4170
|
if (existingEntities.some((e) => e.start <= start && e.end >= end)) continue;
|
|
4119
|
-
if (!
|
|
4171
|
+
if (!contextEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
|
|
4120
4172
|
results.push({
|
|
4121
4173
|
start,
|
|
4122
4174
|
end,
|
|
@@ -4464,6 +4516,7 @@ const applyHotwordRules = (entities, fullText) => {
|
|
|
4464
4516
|
//#region src/filters/boundary-consistency.ts
|
|
4465
4517
|
/** Max gap (in chars) between entities to merge. */
|
|
4466
4518
|
const MAX_GAP = 3;
|
|
4519
|
+
const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
4467
4520
|
/**
|
|
4468
4521
|
* Characters allowed in the gap between two adjacent
|
|
4469
4522
|
* same-label entities that should be merged: spaces,
|
|
@@ -4579,6 +4632,11 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
4579
4632
|
const result = [];
|
|
4580
4633
|
const lastByLabel = /* @__PURE__ */ new Map();
|
|
4581
4634
|
for (const entity of sorted) {
|
|
4635
|
+
if (hasLockedBoundary$1(entity)) {
|
|
4636
|
+
const copy = { ...entity };
|
|
4637
|
+
result.push(copy);
|
|
4638
|
+
continue;
|
|
4639
|
+
}
|
|
4582
4640
|
const prev = lastByLabel.get(entity.label);
|
|
4583
4641
|
if (!prev) {
|
|
4584
4642
|
const copy = { ...entity };
|
|
@@ -4586,7 +4644,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
4586
4644
|
lastByLabel.set(entity.label, copy);
|
|
4587
4645
|
continue;
|
|
4588
4646
|
}
|
|
4589
|
-
if (entity.start < prev.end) {
|
|
4647
|
+
if (!hasLockedBoundary$1(prev) && entity.start < prev.end) {
|
|
4590
4648
|
prev.end = Math.max(prev.end, entity.end);
|
|
4591
4649
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
4592
4650
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -4605,7 +4663,7 @@ const mergeAdjacent = (entities, fullText) => {
|
|
|
4605
4663
|
break;
|
|
4606
4664
|
}
|
|
4607
4665
|
}
|
|
4608
|
-
if (!gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
4666
|
+
if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
|
|
4609
4667
|
prev.end = entity.end;
|
|
4610
4668
|
prev.text = fullText.slice(prev.start, prev.end);
|
|
4611
4669
|
prev.score = Math.max(prev.score, entity.score);
|
|
@@ -4637,6 +4695,7 @@ const fixPartialWords = (entities, fullText) => {
|
|
|
4637
4695
|
})).sort((a, b) => a.entity.end - b.entity.end);
|
|
4638
4696
|
const endPositions = byEnd.map((x) => x.entity.end);
|
|
4639
4697
|
return sorted.map((e, eIdx) => {
|
|
4698
|
+
if (hasLockedBoundary$1(e)) return e;
|
|
4640
4699
|
let newStart = wordStartAt(e.start, boundaries, fullText);
|
|
4641
4700
|
let newEnd = wordEndAt(e.end, boundaries, fullText);
|
|
4642
4701
|
let lo = 0;
|
|
@@ -4731,7 +4790,8 @@ const resolveCrossLabelOverlaps = (entities, fullText) => {
|
|
|
4731
4790
|
if (aContainsB || bContainsA) continue;
|
|
4732
4791
|
const aLen = a.end - a.start;
|
|
4733
4792
|
const bLen = b.end - b.start;
|
|
4734
|
-
|
|
4793
|
+
const aLocked = hasLockedBoundary$1(a);
|
|
4794
|
+
if (aLocked !== hasLockedBoundary$1(b) ? aLocked : a.score > b.score || a.score === b.score && aLen >= bLen) {
|
|
4735
4795
|
b.start = a.end;
|
|
4736
4796
|
b.text = fullText.slice(b.start, b.end);
|
|
4737
4797
|
} else {
|
|
@@ -4759,8 +4819,11 @@ const enforceBoundaryConsistency = (entities, fullText) => {
|
|
|
4759
4819
|
};
|
|
4760
4820
|
//#endregion
|
|
4761
4821
|
//#region src/build-unified-search.ts
|
|
4822
|
+
const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
|
|
4823
|
+
const ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
4762
4824
|
const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
|
|
4763
4825
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
4826
|
+
const customRegexes = config.enableRegex ? config.customRegexes ?? [] : [];
|
|
4764
4827
|
const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
|
|
4765
4828
|
legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
|
|
4766
4829
|
config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
|
|
@@ -4785,12 +4848,21 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4785
4848
|
...datePatterns.map(() => DATE_PATTERN_META),
|
|
4786
4849
|
...signingPatterns.map(() => SIGNING_CLAUSE_META)
|
|
4787
4850
|
];
|
|
4851
|
+
const customRegexMeta = customRegexes.map((entry) => ({
|
|
4852
|
+
label: entry.label,
|
|
4853
|
+
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
|
|
4854
|
+
sourceDetail: "custom-regex"
|
|
4855
|
+
}));
|
|
4788
4856
|
let offset = 0;
|
|
4789
4857
|
const regexSlice = {
|
|
4790
4858
|
start: offset,
|
|
4791
4859
|
end: offset + allRegex.length
|
|
4792
4860
|
};
|
|
4793
4861
|
offset = regexSlice.end;
|
|
4862
|
+
const customRegexSlice = {
|
|
4863
|
+
start: 0,
|
|
4864
|
+
end: customRegexes.length
|
|
4865
|
+
};
|
|
4794
4866
|
const legalFormsSlice = {
|
|
4795
4867
|
start: offset,
|
|
4796
4868
|
end: offset + legalForms.length
|
|
@@ -4811,6 +4883,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4811
4883
|
...triggerEntries
|
|
4812
4884
|
];
|
|
4813
4885
|
const tsRegex = new (getTextSearch())(regexAllPatterns);
|
|
4886
|
+
const tsCustomRegex = new (getTextSearch())(customRegexes.map((entry) => entry.pattern), { overlapStrategy: "all" });
|
|
4814
4887
|
offset = 0;
|
|
4815
4888
|
const denyListOriginals = denyListData?.originals ?? [];
|
|
4816
4889
|
const denyListSlice = {
|
|
@@ -4829,24 +4902,31 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4829
4902
|
end: offset + (gazResult?.patterns.length ?? 0)
|
|
4830
4903
|
};
|
|
4831
4904
|
offset = gazetteerSlice.end;
|
|
4832
|
-
const wrapWholeWord = (s) => ({
|
|
4905
|
+
const wrapWholeWord = (s, wholeWords) => ({
|
|
4833
4906
|
pattern: s,
|
|
4834
4907
|
literal: true,
|
|
4835
|
-
wholeWords
|
|
4908
|
+
wholeWords
|
|
4836
4909
|
});
|
|
4910
|
+
const customDenyListNeedsWholeWords = (pattern) => {
|
|
4911
|
+
const first = pattern.at(0) ?? "";
|
|
4912
|
+
const last = pattern.at(-1) ?? "";
|
|
4913
|
+
return ALNUM_RE.test(first) && ALNUM_RE.test(last);
|
|
4914
|
+
};
|
|
4837
4915
|
const literalAllPatterns = [
|
|
4838
|
-
...denyListOriginals.map(wrapWholeWord),
|
|
4839
|
-
...streetTypes.map(wrapWholeWord),
|
|
4916
|
+
...denyListOriginals.map((pattern, index) => wrapWholeWord(pattern, (denyListData?.sources[index] ?? []).includes("custom-deny-list") ? customDenyListNeedsWholeWords(pattern) : true)),
|
|
4917
|
+
...streetTypes.map((pattern) => wrapWholeWord(pattern, true)),
|
|
4840
4918
|
...gazResult?.patterns ?? []
|
|
4841
4919
|
];
|
|
4842
4920
|
return {
|
|
4843
4921
|
tsRegex,
|
|
4922
|
+
tsCustomRegex,
|
|
4844
4923
|
tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
|
|
4845
4924
|
caseInsensitive: true,
|
|
4846
4925
|
overlapStrategy: "all"
|
|
4847
4926
|
}) : new (getTextSearch())([]),
|
|
4848
4927
|
slices: {
|
|
4849
4928
|
regex: regexSlice,
|
|
4929
|
+
customRegex: customRegexSlice,
|
|
4850
4930
|
legalForms: legalFormsSlice,
|
|
4851
4931
|
triggers: triggersSlice,
|
|
4852
4932
|
denyList: denyListSlice,
|
|
@@ -4854,6 +4934,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4854
4934
|
gazetteer: gazetteerSlice
|
|
4855
4935
|
},
|
|
4856
4936
|
regexMeta,
|
|
4937
|
+
customRegexMeta,
|
|
4857
4938
|
triggerRules: triggers.rules,
|
|
4858
4939
|
denyListData,
|
|
4859
4940
|
gazetteerData: gazResult?.data ?? null
|
|
@@ -4863,9 +4944,11 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
|
|
|
4863
4944
|
//#region src/unified-search.ts
|
|
4864
4945
|
const runUnifiedSearch = (instance, fullText) => {
|
|
4865
4946
|
const regexMatches = instance.tsRegex.findIter(fullText);
|
|
4947
|
+
const customRegexMatches = instance.tsCustomRegex.findIter(fullText);
|
|
4866
4948
|
const normalized = normalizeForSearch(fullText);
|
|
4867
4949
|
return {
|
|
4868
4950
|
regexMatches,
|
|
4951
|
+
customRegexMatches,
|
|
4869
4952
|
literalMatches: instance.tsLiterals.findIter(normalized)
|
|
4870
4953
|
};
|
|
4871
4954
|
};
|
|
@@ -4978,9 +5061,13 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
|
|
|
4978
5061
|
* so the containment rule trusts their length.
|
|
4979
5062
|
*/
|
|
4980
5063
|
const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
|
|
5064
|
+
const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
|
|
5065
|
+
const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
|
|
4981
5066
|
const shouldReplace = (a, b) => {
|
|
4982
5067
|
const aLen = a.end - a.start;
|
|
4983
5068
|
const bLen = b.end - b.start;
|
|
5069
|
+
const aCallerOwned = isCallerOwnedEntity(a);
|
|
5070
|
+
if (aCallerOwned !== isCallerOwnedEntity(b)) return aCallerOwned;
|
|
4984
5071
|
if (a.label === b.label && LITERAL_SOURCES.has(a.source) && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
|
|
4985
5072
|
if (a.label === b.label && LITERAL_SOURCES.has(b.source) && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
|
|
4986
5073
|
const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
|
|
@@ -4992,6 +5079,7 @@ const shouldReplace = (a, b) => {
|
|
|
4992
5079
|
const COLON_LABELS = new Set(["ip address", "mac address"]);
|
|
4993
5080
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
4994
5081
|
const sanitizeEntities = (entities) => entities.flatMap((e) => {
|
|
5082
|
+
if (hasLockedBoundary(e)) return [e];
|
|
4995
5083
|
const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
|
|
4996
5084
|
const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
|
|
4997
5085
|
const lead = e.text.length - leadTrimmed.length;
|
|
@@ -5019,8 +5107,30 @@ const mergeAndDedup = (...layers) => {
|
|
|
5019
5107
|
const entity = sorted[i];
|
|
5020
5108
|
const last = merged[merged.length - 1];
|
|
5021
5109
|
if (!entity || !last) continue;
|
|
5022
|
-
if (last.end <= entity.start)
|
|
5023
|
-
|
|
5110
|
+
if (last.end <= entity.start) {
|
|
5111
|
+
merged.push({ ...entity });
|
|
5112
|
+
continue;
|
|
5113
|
+
}
|
|
5114
|
+
const overlapStart = (() => {
|
|
5115
|
+
for (let j = merged.length - 1; j >= 0; j--) {
|
|
5116
|
+
const existing = merged[j];
|
|
5117
|
+
if (!existing || existing.end <= entity.start) return j + 1;
|
|
5118
|
+
}
|
|
5119
|
+
return 0;
|
|
5120
|
+
})();
|
|
5121
|
+
const overlaps = merged.slice(overlapStart);
|
|
5122
|
+
if (!overlaps.some((existing) => existing.start !== entity.start || existing.end !== entity.end)) {
|
|
5123
|
+
const sameLabelIndex = overlaps.findIndex((existing) => existing.label === entity.label);
|
|
5124
|
+
if (sameLabelIndex === -1) {
|
|
5125
|
+
merged.push({ ...entity });
|
|
5126
|
+
continue;
|
|
5127
|
+
}
|
|
5128
|
+
const actualIndex = overlapStart + sameLabelIndex;
|
|
5129
|
+
const sameLabel = merged[actualIndex];
|
|
5130
|
+
if (sameLabel && shouldReplace(entity, sameLabel)) merged[actualIndex] = { ...entity };
|
|
5131
|
+
continue;
|
|
5132
|
+
}
|
|
5133
|
+
if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
|
|
5024
5134
|
}
|
|
5025
5135
|
return sanitizeEntities(merged);
|
|
5026
5136
|
};
|
|
@@ -5039,7 +5149,7 @@ const getAmountWordsRe = async () => {
|
|
|
5039
5149
|
return amountWordsRe;
|
|
5040
5150
|
};
|
|
5041
5151
|
const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) => {
|
|
5042
|
-
if (e.label !== "monetary amount") return e;
|
|
5152
|
+
if (e.label !== "monetary amount" || isCallerOwnedEntity(e)) return e;
|
|
5043
5153
|
const after = fullText.slice(e.end);
|
|
5044
5154
|
const m = re.exec(after);
|
|
5045
5155
|
if (!m) return e;
|
|
@@ -5052,6 +5162,7 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
|
|
|
5052
5162
|
});
|
|
5053
5163
|
const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
|
|
5054
5164
|
const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
|
|
5165
|
+
const DEFAULT_CUSTOM_REGEX_SCORE = .9;
|
|
5055
5166
|
const filterAllowedLabels = (entities, allowedLabels) => {
|
|
5056
5167
|
if (!allowedLabels) return entities;
|
|
5057
5168
|
return entities.filter((e) => allowedLabels.has(e.label));
|
|
@@ -5066,8 +5177,18 @@ const checkAbort = (signal) => {
|
|
|
5066
5177
|
};
|
|
5067
5178
|
const configKey = (config, gazetteerEntries) => {
|
|
5068
5179
|
const legalFormsEnabled = isLegalFormsEnabled(config);
|
|
5180
|
+
const customDenyFingerprint = config.enableDenyList && config.customDenyList ? config.customDenyList.map((entry) => JSON.stringify({
|
|
5181
|
+
label: entry.label,
|
|
5182
|
+
value: entry.value,
|
|
5183
|
+
variants: [...entry.variants ?? []].sort()
|
|
5184
|
+
})).sort().join("\n") : "";
|
|
5185
|
+
const customRegexFingerprint = config.enableRegex && config.customRegexes ? config.customRegexes.map((entry) => JSON.stringify({
|
|
5186
|
+
label: entry.label,
|
|
5187
|
+
pattern: entry.pattern,
|
|
5188
|
+
score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
|
|
5189
|
+
})).sort().join("\n") : "";
|
|
5069
5190
|
const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
|
|
5070
|
-
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
5191
|
+
return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
|
|
5071
5192
|
};
|
|
5072
5193
|
/**
|
|
5073
5194
|
* Get or build a cached search instance. Cache state
|
|
@@ -5150,10 +5271,11 @@ const runPipeline = async (options) => {
|
|
|
5150
5271
|
const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
|
|
5151
5272
|
const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
|
|
5152
5273
|
checkAbort(signal);
|
|
5153
|
-
const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
5274
|
+
const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
|
|
5154
5275
|
const { slices } = search;
|
|
5155
5276
|
const rawRegexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
|
|
5156
|
-
const
|
|
5277
|
+
const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
|
|
5278
|
+
const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
|
|
5157
5279
|
if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
|
|
5158
5280
|
const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
|
|
5159
5281
|
const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
|
|
@@ -5178,18 +5300,33 @@ const runPipeline = async (options) => {
|
|
|
5178
5300
|
const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
|
|
5179
5301
|
if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
|
|
5180
5302
|
checkAbort(signal);
|
|
5303
|
+
const rawCustomDenyListEntities = rawDenyListEntities.filter((entity) => isCallerOwnedEntity(entity));
|
|
5304
|
+
const rawCuratedDenyListEntities = rawDenyListEntities.filter((entity) => !isCallerOwnedEntity(entity));
|
|
5305
|
+
const customDenyListEntities = filterAllowedLabels(rawCustomDenyListEntities, preHotwordAllowedLabels);
|
|
5181
5306
|
const ruleContextEntities = [
|
|
5182
5307
|
...rawTriggerEntities,
|
|
5183
5308
|
...rawRegexEntities,
|
|
5309
|
+
...customRegexEntities,
|
|
5184
5310
|
...rawLegalFormEntities,
|
|
5185
5311
|
...rawNameCorpusEntities,
|
|
5186
|
-
...
|
|
5312
|
+
...rawCuratedDenyListEntities,
|
|
5313
|
+
...customDenyListEntities,
|
|
5314
|
+
...rawGazetteerEntities
|
|
5315
|
+
];
|
|
5316
|
+
const nerMaskEntities = [
|
|
5317
|
+
...rawTriggerEntities,
|
|
5318
|
+
...rawRegexEntities,
|
|
5319
|
+
...customRegexEntities,
|
|
5320
|
+
...rawLegalFormEntities,
|
|
5321
|
+
...rawNameCorpusEntities,
|
|
5322
|
+
...rawCuratedDenyListEntities,
|
|
5323
|
+
...customDenyListEntities,
|
|
5187
5324
|
...rawGazetteerEntities
|
|
5188
5325
|
];
|
|
5189
5326
|
let rawNerEntities = [];
|
|
5190
5327
|
let nerEntities = [];
|
|
5191
5328
|
if (config.enableNer && nerInference) {
|
|
5192
|
-
const maskResult = maskDetectedSpans(fullText,
|
|
5329
|
+
const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
|
|
5193
5330
|
log("ner", "running inference...");
|
|
5194
5331
|
const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
|
|
5195
5332
|
rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
|
|
@@ -5212,7 +5349,12 @@ const runPipeline = async (options) => {
|
|
|
5212
5349
|
if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
|
|
5213
5350
|
checkAbort(signal);
|
|
5214
5351
|
const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
|
|
5215
|
-
const preBoostEntities =
|
|
5352
|
+
const preBoostEntities = (() => {
|
|
5353
|
+
if (!hotwordsActive) return zoneAdjusted;
|
|
5354
|
+
const hotwordCandidates = zoneAdjusted.filter((entity) => !isCallerOwnedEntity(entity));
|
|
5355
|
+
const callerOwnedEntities = zoneAdjusted.filter(isCallerOwnedEntity);
|
|
5356
|
+
return [...filterAllowedLabels(applyHotwordRules(hotwordCandidates, fullText), allowedLabels), ...callerOwnedEntities];
|
|
5357
|
+
})();
|
|
5216
5358
|
let allEntities;
|
|
5217
5359
|
if (config.enableConfidenceBoost) {
|
|
5218
5360
|
allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
|
|
@@ -5247,7 +5389,7 @@ const runPipeline = async (options) => {
|
|
|
5247
5389
|
checkAbort(signal);
|
|
5248
5390
|
ctx.corefSourceMap.clear();
|
|
5249
5391
|
if (config.enableCoreference) {
|
|
5250
|
-
const terms = await extractDefinedTerms(fullText, merged, ctx);
|
|
5392
|
+
const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
|
|
5251
5393
|
if (terms.length > 0) {
|
|
5252
5394
|
log("coreference", `${terms.length} defined terms`);
|
|
5253
5395
|
const corefSpans = findCoreferenceSpans(fullText, terms, ctx);
|