@stll/anonymize-wasm 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/wasm.mjs CHANGED
@@ -2021,19 +2021,21 @@ const processRegexMatches = (allMatches, sliceStart, sliceEnd, meta_) => {
2021
2021
  if (idx < sliceStart || idx >= sliceEnd) continue;
2022
2022
  const meta = meta_[idx - sliceStart];
2023
2023
  if (!meta) continue;
2024
- if (meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2024
+ if (meta.sourceDetail !== "custom-regex" && meta.label === "phone number" && match.text.length < MIN_PHONE_LENGTH) continue;
2025
2025
  if (meta.validator) {
2026
2026
  const compacted = meta.validator.compact(match.text);
2027
2027
  if (!meta.validator.validate(compacted).valid) continue;
2028
2028
  }
2029
- results.push({
2029
+ const entity = {
2030
2030
  start: match.start,
2031
2031
  end: match.end,
2032
2032
  label: meta.label,
2033
2033
  text: match.text,
2034
2034
  score: meta.score,
2035
2035
  source: DETECTION_SOURCES.REGEX
2036
- });
2036
+ };
2037
+ if (meta.sourceDetail) entity.sourceDetail = meta.sourceDetail;
2038
+ results.push(entity);
2037
2039
  }
2038
2040
  return results;
2039
2041
  };
@@ -3055,6 +3057,7 @@ const loadGenericRoles = (ctx = defaultContext) => {
3055
3057
  };
3056
3058
  /** Sync accessor — returns empty set before init. */
3057
3059
  const getGenericRoles = (ctx) => ctx.genericRoles ?? EMPTY_GENERIC_ROLES;
3060
+ const isCallerOwnedEntity$3 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3058
3061
  /**
3059
3062
  * Filter out entities that are likely false positives:
3060
3063
  * template placeholders, clause/section numbers,
@@ -3067,9 +3070,17 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
3067
3070
  const filtered = [];
3068
3071
  const roles = getGenericRoles(ctx);
3069
3072
  for (const entity of entities) {
3073
+ if (isCallerOwnedEntity$3(entity)) {
3074
+ filtered.push(entity);
3075
+ continue;
3076
+ }
3070
3077
  const normalized = normalizeEntity(entity);
3071
3078
  if (!normalized) continue;
3072
3079
  const trimmed = normalized.text;
3080
+ if (isCallerOwnedEntity$3(normalized)) {
3081
+ filtered.push(normalized);
3082
+ continue;
3083
+ }
3073
3084
  if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
3074
3085
  const maxLen = MAX_ENTITY_LENGTH[normalized.label];
3075
3086
  if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
@@ -3244,6 +3255,7 @@ const EMPTY_PERSON_STOPWORDS = /* @__PURE__ */ new Set();
3244
3255
  /** Sync accessor — returns empty set before init. */
3245
3256
  const getPersonStopwords = (ctx) => ctx.personStopwords ?? EMPTY_PERSON_STOPWORDS;
3246
3257
  const PERSON_CHAIN_BREAK_RE = /[!?;:]|,/u;
3258
+ const WORD_CHAR_RE$1 = /[\p{L}\p{N}]/u;
3247
3259
  const isInitialContinuationGap = (text, gap) => /^\p{Lu}$/u.test(text) && /^\.[^\S\n]{1,2}$/u.test(gap) || /^[^\S\n]{1,2}(?:\p{Lu}\.[^\S\n]{1,2})+$/u.test(gap);
3248
3260
  /**
3249
3261
  * Resolve which dictionaries to load based on country
@@ -3266,27 +3278,31 @@ const buildDenyList = async (config, ctx = defaultContext) => {
3266
3278
  const dictionaries = config.dictionaries;
3267
3279
  const hasDenyList = dictionaries?.denyList && dictionaries?.denyListMeta;
3268
3280
  const hasCities = dictionaries?.cities && dictionaries.cities.length > 0;
3269
- if (!hasDenyList && !hasCities) return buildNameCorpusOnly(config, ctx);
3281
+ const hasCustomDenyList = config.customDenyList !== void 0 && config.customDenyList.length > 0;
3282
+ if (!hasDenyList && !hasCities && !hasCustomDenyList) return buildNameCorpusOnly(config, ctx);
3270
3283
  const allowedCountries = resolveCountries(config.denyListRegions, config.denyListCountries);
3271
3284
  const excluded = config.denyListExcludeCategories;
3272
3285
  const excludeCategories = excluded ? new Set(excluded) : /* @__PURE__ */ new Set();
3273
3286
  const patternList = [];
3274
3287
  const labelList = [];
3288
+ const customLabelList = [];
3275
3289
  const sourceList = [];
3276
3290
  const patternIndex = /* @__PURE__ */ new Map();
3277
- const addDenyListEntry = (entry, label) => {
3278
- const normalized = normalizeForSearch(entry).replace(/[|\\]/g, "");
3291
+ const addDenyListEntry = (entry, label, source = "deny-list") => {
3292
+ const normalized = source === "custom-deny-list" ? normalizeForSearch(entry) : normalizeForSearch(entry).replace(/[|\\]/g, "");
3279
3293
  if (normalized.length === 0) return;
3280
3294
  const lower = normalized.toLowerCase();
3281
3295
  const existing = patternIndex.get(lower);
3282
3296
  if (existing !== void 0) {
3283
3297
  if (!labelList[existing].includes(label)) labelList[existing].push(label);
3284
- if (!sourceList[existing].includes("deny-list")) sourceList[existing].push("deny-list");
3298
+ if (!sourceList[existing].includes(source)) sourceList[existing].push(source);
3299
+ if (source === "custom-deny-list" && !customLabelList[existing].includes(label)) customLabelList[existing].push(label);
3285
3300
  } else {
3286
3301
  patternIndex.set(lower, patternList.length);
3287
3302
  patternList.push(normalized);
3288
3303
  labelList.push([label]);
3289
- sourceList.push(["deny-list"]);
3304
+ customLabelList.push(source === "custom-deny-list" ? [label] : []);
3305
+ sourceList.push([source]);
3290
3306
  }
3291
3307
  };
3292
3308
  if (hasDenyList) {
@@ -3304,10 +3320,15 @@ const buildDenyList = async (config, ctx = defaultContext) => {
3304
3320
  }
3305
3321
  }
3306
3322
  if (hasCities && !excludeCategories.has("Places")) for (const entry of dictionaries.cities) addDenyListEntry(entry, "address");
3323
+ if (hasCustomDenyList) for (const entry of config.customDenyList) {
3324
+ addDenyListEntry(entry.value, entry.label, "custom-deny-list");
3325
+ for (const variant of entry.variants ?? []) addDenyListEntry(variant, entry.label, "custom-deny-list");
3326
+ }
3307
3327
  appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, patternIndex);
3308
3328
  if (patternList.length === 0) return null;
3309
3329
  return {
3310
3330
  labels: labelList,
3331
+ customLabels: customLabelList,
3311
3332
  originals: patternList,
3312
3333
  sources: sourceList
3313
3334
  };
@@ -3324,11 +3345,13 @@ const buildNameCorpusOnly = (config, ctx) => {
3324
3345
  if ((excluded ? new Set(excluded) : /* @__PURE__ */ new Set()).has("Names")) return null;
3325
3346
  const patternList = [];
3326
3347
  const labelList = [];
3348
+ const customLabelList = [];
3327
3349
  const sourceList = [];
3328
3350
  appendNameCorpusEntries(config, ctx, patternList, labelList, sourceList, /* @__PURE__ */ new Map());
3329
3351
  if (patternList.length === 0) return null;
3330
3352
  return {
3331
3353
  labels: labelList,
3354
+ customLabels: customLabelList,
3332
3355
  originals: patternList,
3333
3356
  sources: sourceList
3334
3357
  };
@@ -3374,6 +3397,14 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
3374
3397
  }
3375
3398
  }
3376
3399
  };
3400
+ const customMatchHasValidEdges = (fullText, start, end, pattern) => {
3401
+ if (!WORD_CHAR_RE$1.test(pattern)) return true;
3402
+ const prev = fullText[start - 1] ?? "";
3403
+ const next = fullText[end] ?? "";
3404
+ if (WORD_CHAR_RE$1.test(prev)) return false;
3405
+ if (WORD_CHAR_RE$1.test(next)) return false;
3406
+ return true;
3407
+ };
3377
3408
  /**
3378
3409
  * Ensure all deny-list support data (stopwords, allow
3379
3410
  * list, person stopwords, generic roles) is loaded on
@@ -3411,18 +3442,24 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3411
3442
  const idx = match.pattern;
3412
3443
  if (idx < sliceStart || idx >= sliceEnd) continue;
3413
3444
  const localIdx = idx - sliceStart;
3414
- const sourceChar = fullText[match.start] ?? "";
3415
- if (!UPPER_START_RE.test(sourceChar)) continue;
3445
+ const sources = data.sources[localIdx] ?? [];
3416
3446
  const matchText = fullText.slice(match.start, match.end);
3447
+ const sourceChar = fullText[match.start] ?? "";
3417
3448
  const keyword = matchText.toLowerCase();
3418
- if (getStopwords(ctx).has(keyword) || getAllowList(ctx).has(keyword)) continue;
3419
- if (ALL_UPPER_RE.test(matchText)) continue;
3420
3449
  const labels = data.labels[localIdx];
3421
- if (!labels || labels.length === 0) continue;
3450
+ const pattern = data.originals[localIdx] ?? "";
3451
+ const customPatternLabels = data.customLabels[localIdx] ?? [];
3452
+ const customEdgesAreValid = customMatchHasValidEdges(fullText, match.start, match.end, pattern);
3453
+ const customLabels = customEdgesAreValid ? customPatternLabels : [];
3454
+ if ((!labels || labels.length === 0) && customLabels.length === 0) continue;
3455
+ const curatedLabels = UPPER_START_RE.test(sourceChar) && !getStopwords(ctx).has(keyword) && !getAllowList(ctx).has(keyword) && !ALL_UPPER_RE.test(matchText) ? (labels ?? []).filter((label) => !customPatternLabels.includes(label) && customEdgesAreValid) : [];
3456
+ if (curatedLabels.length === 0 && customLabels.length === 0) continue;
3422
3457
  const entry = {
3423
3458
  start: match.start,
3424
3459
  end: match.end,
3425
- labels,
3460
+ labels: curatedLabels,
3461
+ customLabels,
3462
+ sources,
3426
3463
  text: matchText,
3427
3464
  patternIdx: localIdx
3428
3465
  };
@@ -3433,22 +3470,31 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
3433
3470
  const results = [];
3434
3471
  const nameHits = [];
3435
3472
  for (const [, matches] of Array.from(matchesByPattern)) {
3436
- const first = matches[0];
3437
- if (!first) continue;
3438
- const hasPerson = first.labels.includes("person");
3439
- const nonPersonLabels = first.labels.filter((l) => l !== "person");
3440
- if (hasPerson) {
3441
- const keyword = first.text.toLowerCase();
3442
- if (!getPersonStopwords(ctx).has(keyword)) for (const m of matches) nameHits.push(m);
3443
- }
3444
- for (const m of matches) for (const label of nonPersonLabels) results.push({
3473
+ if (!matches[0]) continue;
3474
+ for (const m of matches) for (const label of m.customLabels) results.push({
3445
3475
  start: m.start,
3446
3476
  end: m.end,
3447
3477
  label,
3448
3478
  text: m.text,
3449
3479
  score: .9,
3450
- source: DETECTION_SOURCES.DENY_LIST
3480
+ source: DETECTION_SOURCES.DENY_LIST,
3481
+ sourceDetail: "custom-deny-list"
3451
3482
  });
3483
+ for (const m of matches) {
3484
+ if (m.labels.includes("person")) {
3485
+ const keyword = m.text.toLowerCase();
3486
+ if (!getPersonStopwords(ctx).has(keyword)) nameHits.push(m);
3487
+ }
3488
+ const nonPersonLabels = m.labels.filter((l) => l !== "person");
3489
+ for (const label of nonPersonLabels) results.push({
3490
+ start: m.start,
3491
+ end: m.end,
3492
+ label,
3493
+ text: m.text,
3494
+ score: .9,
3495
+ source: DETECTION_SOURCES.DENY_LIST
3496
+ });
3497
+ }
3452
3498
  }
3453
3499
  nameHits.sort((a, b) => a.start - b.start);
3454
3500
  const nameConsumed = /* @__PURE__ */ new Set();
@@ -3528,6 +3574,7 @@ const TRAILING_WORD_EXCLUSIONS = new Set([
3528
3574
  const extendCityDistricts = (entities, fullText) => {
3529
3575
  for (const entity of entities) {
3530
3576
  if (entity.label !== "address") continue;
3577
+ if (entity.sourceDetail === "custom-deny-list") continue;
3531
3578
  const afterMatch = fullText.slice(entity.end);
3532
3579
  const suffixM = DISTRICT_SUFFIX_RE.exec(afterMatch);
3533
3580
  if (suffixM) {
@@ -3646,6 +3693,7 @@ const collectSeeds = (allMatches, sliceStart, sliceEnd, fullText, existingEntiti
3646
3693
  }
3647
3694
  for (const e of existingEntities) {
3648
3695
  if (e.label !== "address") continue;
3696
+ if (e.sourceDetail === "custom-deny-list") continue;
3649
3697
  if (e.source === "deny-list") seeds.push({
3650
3698
  type: "city",
3651
3699
  start: e.start,
@@ -3812,6 +3860,7 @@ const processAddressSeeds = async (allMatches, sliceStart, sliceEnd, fullText, e
3812
3860
  const TRAILING_SEP = /[,\s]+$/;
3813
3861
  const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
3814
3862
  const ORG_PROPAGATION_SCORE = .9;
3863
+ const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3815
3864
  /**
3816
3865
  * After the main detection pass, collect organization
3817
3866
  * entities with a legal form suffix, strip the suffix
@@ -3824,6 +3873,7 @@ const propagateOrgNames = (entities, fullText) => {
3824
3873
  const seenBases = /* @__PURE__ */ new Set();
3825
3874
  for (const e of entities) {
3826
3875
  if (e.label !== "organization") continue;
3876
+ if (isCallerOwnedEntity$2(e)) continue;
3827
3877
  for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
3828
3878
  const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
3829
3879
  if (base.length >= 3 && !seenBases.has(base)) {
@@ -3903,6 +3953,7 @@ const NEAR_MISS_BAND = .15;
3903
3953
  const BOOST_PER_NEIGHBOUR = .05;
3904
3954
  const CONTEXT_WINDOW_CHARS = 150;
3905
3955
  const HIGH_CONFIDENCE_FLOOR = .9;
3956
+ const isCallerOwnedEntity$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
3906
3957
  /**
3907
3958
  * Boost confidence of near-miss NER entities that appear
3908
3959
  * near high-confidence detections (regex, trigger phrase).
@@ -4011,7 +4062,7 @@ const getStreetAbbrevs = () => _streetAbbrevs ?? /* @__PURE__ */ new Set();
4011
4062
  */
4012
4063
  const detectStreetPatternsNearAddresses = (fullText, existingEntities) => {
4013
4064
  const results = [];
4014
- const addressEntities = existingEntities.filter((e) => e.label === "address");
4065
+ const addressEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity))).filter((e) => e.label === "address");
4015
4066
  const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
4016
4067
  const houseNumRe = /\b(?:\d{1,4}\/\d+[a-zA-Z]\b|\d{3,4}\/\d+\b|(?:1[3-9]|[2-9]\d)\/\d{3,}\b)/g;
4017
4068
  houseNumRe.lastIndex = 0;
@@ -4107,6 +4158,7 @@ const ORPHAN_STREET_RE = /^\s*(\p{Lu}[\p{Ll}\p{Lu}]+(?:\s+[\p{Lu}\p{Ll}][\p{Ll}]
4107
4158
  */
4108
4159
  const detectOrphanStreetLines = (fullText, existingEntities) => {
4109
4160
  const headerEnd = Math.floor(fullText.length * HEADER_ZONE_FRACTION);
4161
+ const contextEntities = existingEntities.filter((entity) => !(entity.label === "address" && isCallerOwnedEntity$1(entity)));
4110
4162
  const results = [];
4111
4163
  ORPHAN_STREET_RE.lastIndex = 0;
4112
4164
  for (let m = ORPHAN_STREET_RE.exec(fullText); m !== null; m = ORPHAN_STREET_RE.exec(fullText)) {
@@ -4116,7 +4168,7 @@ const detectOrphanStreetLines = (fullText, existingEntities) => {
4116
4168
  const end = start + captured.length;
4117
4169
  if (start >= headerEnd) continue;
4118
4170
  if (existingEntities.some((e) => e.start <= start && e.end >= end)) continue;
4119
- if (!existingEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
4171
+ if (!contextEntities.some((e) => Math.abs(e.start - end) < 200 || Math.abs(e.end - start) < 200)) continue;
4120
4172
  results.push({
4121
4173
  start,
4122
4174
  end,
@@ -4464,6 +4516,7 @@ const applyHotwordRules = (entities, fullText) => {
4464
4516
  //#region src/filters/boundary-consistency.ts
4465
4517
  /** Max gap (in chars) between entities to merge. */
4466
4518
  const MAX_GAP = 3;
4519
+ const hasLockedBoundary$1 = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
4467
4520
  /**
4468
4521
  * Characters allowed in the gap between two adjacent
4469
4522
  * same-label entities that should be merged: spaces,
@@ -4579,6 +4632,11 @@ const mergeAdjacent = (entities, fullText) => {
4579
4632
  const result = [];
4580
4633
  const lastByLabel = /* @__PURE__ */ new Map();
4581
4634
  for (const entity of sorted) {
4635
+ if (hasLockedBoundary$1(entity)) {
4636
+ const copy = { ...entity };
4637
+ result.push(copy);
4638
+ continue;
4639
+ }
4582
4640
  const prev = lastByLabel.get(entity.label);
4583
4641
  if (!prev) {
4584
4642
  const copy = { ...entity };
@@ -4586,7 +4644,7 @@ const mergeAdjacent = (entities, fullText) => {
4586
4644
  lastByLabel.set(entity.label, copy);
4587
4645
  continue;
4588
4646
  }
4589
- if (entity.start < prev.end) {
4647
+ if (!hasLockedBoundary$1(prev) && entity.start < prev.end) {
4590
4648
  prev.end = Math.max(prev.end, entity.end);
4591
4649
  prev.text = fullText.slice(prev.start, prev.end);
4592
4650
  prev.score = Math.max(prev.score, entity.score);
@@ -4605,7 +4663,7 @@ const mergeAdjacent = (entities, fullText) => {
4605
4663
  break;
4606
4664
  }
4607
4665
  }
4608
- if (!gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
4666
+ if (!hasLockedBoundary$1(prev) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
4609
4667
  prev.end = entity.end;
4610
4668
  prev.text = fullText.slice(prev.start, prev.end);
4611
4669
  prev.score = Math.max(prev.score, entity.score);
@@ -4637,6 +4695,7 @@ const fixPartialWords = (entities, fullText) => {
4637
4695
  })).sort((a, b) => a.entity.end - b.entity.end);
4638
4696
  const endPositions = byEnd.map((x) => x.entity.end);
4639
4697
  return sorted.map((e, eIdx) => {
4698
+ if (hasLockedBoundary$1(e)) return e;
4640
4699
  let newStart = wordStartAt(e.start, boundaries, fullText);
4641
4700
  let newEnd = wordEndAt(e.end, boundaries, fullText);
4642
4701
  let lo = 0;
@@ -4731,7 +4790,8 @@ const resolveCrossLabelOverlaps = (entities, fullText) => {
4731
4790
  if (aContainsB || bContainsA) continue;
4732
4791
  const aLen = a.end - a.start;
4733
4792
  const bLen = b.end - b.start;
4734
- if (a.score > b.score || a.score === b.score && aLen >= bLen) {
4793
+ const aLocked = hasLockedBoundary$1(a);
4794
+ if (aLocked !== hasLockedBoundary$1(b) ? aLocked : a.score > b.score || a.score === b.score && aLen >= bLen) {
4735
4795
  b.start = a.end;
4736
4796
  b.text = fullText.slice(b.start, b.end);
4737
4797
  } else {
@@ -4759,8 +4819,11 @@ const enforceBoundaryConsistency = (entities, fullText) => {
4759
4819
  };
4760
4820
  //#endregion
4761
4821
  //#region src/build-unified-search.ts
4822
+ const DEFAULT_CUSTOM_REGEX_SCORE$1 = .9;
4823
+ const ALNUM_RE = /[\p{L}\p{N}]/u;
4762
4824
  const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultContext) => {
4763
4825
  const legalFormsEnabled = isLegalFormsEnabled(config);
4826
+ const customRegexes = config.enableRegex ? config.customRegexes ?? [] : [];
4764
4827
  const [legalForms, triggers, denyListData, streetTypes, currencyPatterns, datePatterns, signingPatterns] = await Promise.all([
4765
4828
  legalFormsEnabled ? buildLegalFormPatterns() : Promise.resolve([]),
4766
4829
  config.enableTriggerPhrases ? buildTriggerPatterns() : Promise.resolve({
@@ -4785,12 +4848,21 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4785
4848
  ...datePatterns.map(() => DATE_PATTERN_META),
4786
4849
  ...signingPatterns.map(() => SIGNING_CLAUSE_META)
4787
4850
  ];
4851
+ const customRegexMeta = customRegexes.map((entry) => ({
4852
+ label: entry.label,
4853
+ score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE$1,
4854
+ sourceDetail: "custom-regex"
4855
+ }));
4788
4856
  let offset = 0;
4789
4857
  const regexSlice = {
4790
4858
  start: offset,
4791
4859
  end: offset + allRegex.length
4792
4860
  };
4793
4861
  offset = regexSlice.end;
4862
+ const customRegexSlice = {
4863
+ start: 0,
4864
+ end: customRegexes.length
4865
+ };
4794
4866
  const legalFormsSlice = {
4795
4867
  start: offset,
4796
4868
  end: offset + legalForms.length
@@ -4811,6 +4883,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4811
4883
  ...triggerEntries
4812
4884
  ];
4813
4885
  const tsRegex = new (getTextSearch())(regexAllPatterns);
4886
+ const tsCustomRegex = new (getTextSearch())(customRegexes.map((entry) => entry.pattern), { overlapStrategy: "all" });
4814
4887
  offset = 0;
4815
4888
  const denyListOriginals = denyListData?.originals ?? [];
4816
4889
  const denyListSlice = {
@@ -4829,24 +4902,31 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4829
4902
  end: offset + (gazResult?.patterns.length ?? 0)
4830
4903
  };
4831
4904
  offset = gazetteerSlice.end;
4832
- const wrapWholeWord = (s) => ({
4905
+ const wrapWholeWord = (s, wholeWords) => ({
4833
4906
  pattern: s,
4834
4907
  literal: true,
4835
- wholeWords: true
4908
+ wholeWords
4836
4909
  });
4910
+ const customDenyListNeedsWholeWords = (pattern) => {
4911
+ const first = pattern.at(0) ?? "";
4912
+ const last = pattern.at(-1) ?? "";
4913
+ return ALNUM_RE.test(first) && ALNUM_RE.test(last);
4914
+ };
4837
4915
  const literalAllPatterns = [
4838
- ...denyListOriginals.map(wrapWholeWord),
4839
- ...streetTypes.map(wrapWholeWord),
4916
+ ...denyListOriginals.map((pattern, index) => wrapWholeWord(pattern, (denyListData?.sources[index] ?? []).includes("custom-deny-list") ? customDenyListNeedsWholeWords(pattern) : true)),
4917
+ ...streetTypes.map((pattern) => wrapWholeWord(pattern, true)),
4840
4918
  ...gazResult?.patterns ?? []
4841
4919
  ];
4842
4920
  return {
4843
4921
  tsRegex,
4922
+ tsCustomRegex,
4844
4923
  tsLiterals: literalAllPatterns.length > 0 ? new (getTextSearch())(literalAllPatterns, {
4845
4924
  caseInsensitive: true,
4846
4925
  overlapStrategy: "all"
4847
4926
  }) : new (getTextSearch())([]),
4848
4927
  slices: {
4849
4928
  regex: regexSlice,
4929
+ customRegex: customRegexSlice,
4850
4930
  legalForms: legalFormsSlice,
4851
4931
  triggers: triggersSlice,
4852
4932
  denyList: denyListSlice,
@@ -4854,6 +4934,7 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4854
4934
  gazetteer: gazetteerSlice
4855
4935
  },
4856
4936
  regexMeta,
4937
+ customRegexMeta,
4857
4938
  triggerRules: triggers.rules,
4858
4939
  denyListData,
4859
4940
  gazetteerData: gazResult?.data ?? null
@@ -4863,9 +4944,11 @@ const buildUnifiedSearch = async (config, gazetteerEntries = [], ctx = defaultCo
4863
4944
  //#region src/unified-search.ts
4864
4945
  const runUnifiedSearch = (instance, fullText) => {
4865
4946
  const regexMatches = instance.tsRegex.findIter(fullText);
4947
+ const customRegexMatches = instance.tsCustomRegex.findIter(fullText);
4866
4948
  const normalized = normalizeForSearch(fullText);
4867
4949
  return {
4868
4950
  regexMatches,
4951
+ customRegexMatches,
4869
4952
  literalMatches: instance.tsLiterals.findIter(normalized)
4870
4953
  };
4871
4954
  };
@@ -4978,9 +5061,13 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
4978
5061
  * so the containment rule trusts their length.
4979
5062
  */
4980
5063
  const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
5064
+ const isCallerOwnedEntity = (entity) => entity.sourceDetail === "custom-deny-list" || entity.sourceDetail === "custom-regex";
5065
+ const hasLockedBoundary = (entity) => isCallerOwnedEntity(entity);
4981
5066
  const shouldReplace = (a, b) => {
4982
5067
  const aLen = a.end - a.start;
4983
5068
  const bLen = b.end - b.start;
5069
+ const aCallerOwned = isCallerOwnedEntity(a);
5070
+ if (aCallerOwned !== isCallerOwnedEntity(b)) return aCallerOwned;
4984
5071
  if (a.label === b.label && LITERAL_SOURCES.has(a.source) && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
4985
5072
  if (a.label === b.label && LITERAL_SOURCES.has(b.source) && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
4986
5073
  const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
@@ -4992,6 +5079,7 @@ const shouldReplace = (a, b) => {
4992
5079
  const COLON_LABELS = new Set(["ip address", "mac address"]);
4993
5080
  /** Strip leading/trailing whitespace and punctuation. */
4994
5081
  const sanitizeEntities = (entities) => entities.flatMap((e) => {
5082
+ if (hasLockedBoundary(e)) return [e];
4995
5083
  const strip = COLON_LABELS.has(e.label) ? /[\s,;]+/ : /[\s:,;]+/;
4996
5084
  const leadTrimmed = e.text.replace(/^(?:\.\s)+/, "").replace(new RegExp(`^${strip.source}`, strip.flags), "");
4997
5085
  const lead = e.text.length - leadTrimmed.length;
@@ -5019,8 +5107,30 @@ const mergeAndDedup = (...layers) => {
5019
5107
  const entity = sorted[i];
5020
5108
  const last = merged[merged.length - 1];
5021
5109
  if (!entity || !last) continue;
5022
- if (last.end <= entity.start) merged.push({ ...entity });
5023
- else if (shouldReplace(entity, last)) merged[merged.length - 1] = { ...entity };
5110
+ if (last.end <= entity.start) {
5111
+ merged.push({ ...entity });
5112
+ continue;
5113
+ }
5114
+ const overlapStart = (() => {
5115
+ for (let j = merged.length - 1; j >= 0; j--) {
5116
+ const existing = merged[j];
5117
+ if (!existing || existing.end <= entity.start) return j + 1;
5118
+ }
5119
+ return 0;
5120
+ })();
5121
+ const overlaps = merged.slice(overlapStart);
5122
+ if (!overlaps.some((existing) => existing.start !== entity.start || existing.end !== entity.end)) {
5123
+ const sameLabelIndex = overlaps.findIndex((existing) => existing.label === entity.label);
5124
+ if (sameLabelIndex === -1) {
5125
+ merged.push({ ...entity });
5126
+ continue;
5127
+ }
5128
+ const actualIndex = overlapStart + sameLabelIndex;
5129
+ const sameLabel = merged[actualIndex];
5130
+ if (sameLabel && shouldReplace(entity, sameLabel)) merged[actualIndex] = { ...entity };
5131
+ continue;
5132
+ }
5133
+ if (overlaps.every((existing) => shouldReplace(entity, existing))) merged.splice(overlapStart, overlaps.length, { ...entity });
5024
5134
  }
5025
5135
  return sanitizeEntities(merged);
5026
5136
  };
@@ -5039,7 +5149,7 @@ const getAmountWordsRe = async () => {
5039
5149
  return amountWordsRe;
5040
5150
  };
5041
5151
  const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) => {
5042
- if (e.label !== "monetary amount") return e;
5152
+ if (e.label !== "monetary amount" || isCallerOwnedEntity(e)) return e;
5043
5153
  const after = fullText.slice(e.end);
5044
5154
  const m = re.exec(after);
5045
5155
  if (!m) return e;
@@ -5052,6 +5162,7 @@ const extendMonetaryAmountWords = (entities, fullText, re) => entities.map((e) =
5052
5162
  });
5053
5163
  const createAllowedLabelSetFromLabels = (labels) => labels.length > 0 ? new Set(labels) : null;
5054
5164
  const createAllowedLabelSet = (config) => createAllowedLabelSetFromLabels(config.labels);
5165
+ const DEFAULT_CUSTOM_REGEX_SCORE = .9;
5055
5166
  const filterAllowedLabels = (entities, allowedLabels) => {
5056
5167
  if (!allowedLabels) return entities;
5057
5168
  return entities.filter((e) => allowedLabels.has(e.label));
@@ -5066,8 +5177,18 @@ const checkAbort = (signal) => {
5066
5177
  };
5067
5178
  const configKey = (config, gazetteerEntries) => {
5068
5179
  const legalFormsEnabled = isLegalFormsEnabled(config);
5180
+ const customDenyFingerprint = config.enableDenyList && config.customDenyList ? config.customDenyList.map((entry) => JSON.stringify({
5181
+ label: entry.label,
5182
+ value: entry.value,
5183
+ variants: [...entry.variants ?? []].sort()
5184
+ })).sort().join("\n") : "";
5185
+ const customRegexFingerprint = config.enableRegex && config.customRegexes ? config.customRegexes.map((entry) => JSON.stringify({
5186
+ label: entry.label,
5187
+ pattern: entry.pattern,
5188
+ score: entry.score ?? DEFAULT_CUSTOM_REGEX_SCORE
5189
+ })).sort().join("\n") : "";
5069
5190
  const gazFingerprint = config.enableGazetteer && gazetteerEntries.length > 0 ? gazetteerEntries.map((e) => `${e.id}:${e.canonical}:${e.label}:${[...e.variants].sort().join(",")}`).toSorted().join(";") : "";
5070
- return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${config.enableGazetteer}:${gazFingerprint}`;
5191
+ return `${config.enableDenyList}:${config.enableTriggerPhrases}:${legalFormsEnabled}:${config.enableNameCorpus}:${config.enableRegex}:${config.denyListCountries?.toSorted().join(",") ?? ""}:${config.denyListRegions?.toSorted().join(",") ?? ""}:${config.denyListExcludeCategories?.toSorted().join(",") ?? ""}:${customDenyFingerprint}:${customRegexFingerprint}:${config.enableGazetteer}:${gazFingerprint}`;
5071
5192
  };
5072
5193
  /**
5073
5194
  * Get or build a cached search instance. Cache state
@@ -5150,10 +5271,11 @@ const runPipeline = async (options) => {
5150
5271
  const preHotwordAllowedLabels = hotwordsActive ? createAllowedLabelSetFromLabels(expandLabelsForHotwordRules(config.labels)) : allowedLabels;
5151
5272
  const search = cachedSearch ?? await getCachedSearch(config, gazetteerEntries, ctx);
5152
5273
  checkAbort(signal);
5153
- const { regexMatches, literalMatches } = runUnifiedSearch(search, fullText);
5274
+ const { regexMatches, customRegexMatches, literalMatches } = runUnifiedSearch(search, fullText);
5154
5275
  const { slices } = search;
5155
5276
  const rawRegexEntities = config.enableRegex ? processRegexMatches(regexMatches, slices.regex.start, slices.regex.end, search.regexMeta) : [];
5156
- const regexEntities = filterAllowedLabels(rawRegexEntities, preHotwordAllowedLabels);
5277
+ const customRegexEntities = filterAllowedLabels(config.enableRegex ? processRegexMatches(customRegexMatches, slices.customRegex.start, slices.customRegex.end, search.customRegexMeta) : [], preHotwordAllowedLabels);
5278
+ const regexEntities = filterAllowedLabels([...rawRegexEntities, ...customRegexEntities], preHotwordAllowedLabels);
5157
5279
  if (regexEntities.length > 0) log("regex", `${regexEntities.length} matches`);
5158
5280
  const rawLegalFormEntities = legalFormsEnabled ? processLegalFormMatches(regexMatches, slices.legalForms.start, slices.legalForms.end, fullText) : [];
5159
5281
  const legalFormEntities = filterAllowedLabels(rawLegalFormEntities, preHotwordAllowedLabels);
@@ -5178,18 +5300,33 @@ const runPipeline = async (options) => {
5178
5300
  const gazetteerEntities = filterAllowedLabels(rawGazetteerEntities, preHotwordAllowedLabels);
5179
5301
  if (gazetteerEntities.length > 0) log("gazetteer", `${gazetteerEntities.length} matches`);
5180
5302
  checkAbort(signal);
5303
+ const rawCustomDenyListEntities = rawDenyListEntities.filter((entity) => isCallerOwnedEntity(entity));
5304
+ const rawCuratedDenyListEntities = rawDenyListEntities.filter((entity) => !isCallerOwnedEntity(entity));
5305
+ const customDenyListEntities = filterAllowedLabels(rawCustomDenyListEntities, preHotwordAllowedLabels);
5181
5306
  const ruleContextEntities = [
5182
5307
  ...rawTriggerEntities,
5183
5308
  ...rawRegexEntities,
5309
+ ...customRegexEntities,
5184
5310
  ...rawLegalFormEntities,
5185
5311
  ...rawNameCorpusEntities,
5186
- ...rawDenyListEntities,
5312
+ ...rawCuratedDenyListEntities,
5313
+ ...customDenyListEntities,
5314
+ ...rawGazetteerEntities
5315
+ ];
5316
+ const nerMaskEntities = [
5317
+ ...rawTriggerEntities,
5318
+ ...rawRegexEntities,
5319
+ ...customRegexEntities,
5320
+ ...rawLegalFormEntities,
5321
+ ...rawNameCorpusEntities,
5322
+ ...rawCuratedDenyListEntities,
5323
+ ...customDenyListEntities,
5187
5324
  ...rawGazetteerEntities
5188
5325
  ];
5189
5326
  let rawNerEntities = [];
5190
5327
  let nerEntities = [];
5191
5328
  if (config.enableNer && nerInference) {
5192
- const maskResult = maskDetectedSpans(fullText, ruleContextEntities);
5329
+ const maskResult = maskDetectedSpans(fullText, nerMaskEntities);
5193
5330
  log("ner", "running inference...");
5194
5331
  const rawNer = await nerInference(maskResult.maskedText, [...getRequestedNerLabels(config, hotwordsActive)], config.threshold, signal);
5195
5332
  rawNerEntities = unmaskNerEntities(rawNer, maskResult, fullText);
@@ -5212,7 +5349,12 @@ const runPipeline = async (options) => {
5212
5349
  if (addressSeedEntities.length > 0) log("address-seeds", `${addressSeedEntities.length} expanded`);
5213
5350
  checkAbort(signal);
5214
5351
  const zoneAdjusted = applyZoneAdjustments([...preAddressEntities, ...addressSeedEntities], zones);
5215
- const preBoostEntities = hotwordsActive ? filterAllowedLabels(applyHotwordRules(zoneAdjusted, fullText), allowedLabels) : zoneAdjusted;
5352
+ const preBoostEntities = (() => {
5353
+ if (!hotwordsActive) return zoneAdjusted;
5354
+ const hotwordCandidates = zoneAdjusted.filter((entity) => !isCallerOwnedEntity(entity));
5355
+ const callerOwnedEntities = zoneAdjusted.filter(isCallerOwnedEntity);
5356
+ return [...filterAllowedLabels(applyHotwordRules(hotwordCandidates, fullText), allowedLabels), ...callerOwnedEntities];
5357
+ })();
5216
5358
  let allEntities;
5217
5359
  if (config.enableConfidenceBoost) {
5218
5360
  allEntities = boostNearMissEntities(preBoostEntities, config.threshold);
@@ -5247,7 +5389,7 @@ const runPipeline = async (options) => {
5247
5389
  checkAbort(signal);
5248
5390
  ctx.corefSourceMap.clear();
5249
5391
  if (config.enableCoreference) {
5250
- const terms = await extractDefinedTerms(fullText, merged, ctx);
5392
+ const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
5251
5393
  if (terms.length > 0) {
5252
5394
  log("coreference", `${terms.length} defined terms`);
5253
5395
  const corefSpans = findCoreferenceSpans(fullText, terms, ctx);