@stll/anonymize-wasm 1.4.8 → 1.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/wasm.d.mts +52 -24
- package/dist/wasm.mjs +277 -54
- package/dist/wasm.mjs.map +1 -1
- package/package.json +1 -1
package/dist/wasm.d.mts
CHANGED
|
@@ -5,18 +5,45 @@ import { Tokenizer } from "@huggingface/tokenizers";
|
|
|
5
5
|
|
|
6
6
|
//#region src/types.d.ts
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
* Every detection layer produces these.
|
|
8
|
+
* Fields shared by every entity span in the source text.
|
|
10
9
|
*/
|
|
11
|
-
type
|
|
10
|
+
type EntityBase = {
|
|
12
11
|
start: number;
|
|
13
12
|
end: number;
|
|
14
13
|
label: string;
|
|
15
14
|
text: string;
|
|
16
15
|
score: number;
|
|
17
|
-
source: DetectionSource;
|
|
18
16
|
sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
|
|
19
17
|
};
|
|
18
|
+
/**
|
|
19
|
+
* A PII entity span found by a primary detection layer
|
|
20
|
+
* (regex, NER, legal forms, deny list, ...).
|
|
21
|
+
*/
|
|
22
|
+
type DetectedEntity = EntityBase & {
|
|
23
|
+
source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* An alias mention of a previously detected entity: a
|
|
27
|
+
* defined term ("the Seller") or a propagated bare
|
|
28
|
+
* mention ("Acme" after "Acme Corp.").
|
|
29
|
+
*
|
|
30
|
+
* `corefSourceText` is required by construction, so an
|
|
31
|
+
* alias cannot exist without the link back to its source
|
|
32
|
+
* entity. Placeholder numbering reads it to give the
|
|
33
|
+
* alias the same placeholder as the source. The link
|
|
34
|
+
* travels with the entity instead of living in a
|
|
35
|
+
* side-channel map that a producer could forget to
|
|
36
|
+
* write — or that a later pass could clear.
|
|
37
|
+
*/
|
|
38
|
+
type CorefAliasEntity = EntityBase & {
|
|
39
|
+
source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
|
|
40
|
+
corefSourceText: string;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* A detected PII entity span in the source text.
|
|
44
|
+
* Every detection layer produces these.
|
|
45
|
+
*/
|
|
46
|
+
type Entity = DetectedEntity | CorefAliasEntity;
|
|
20
47
|
/**
|
|
21
48
|
* Entity after human review. Extends the base Entity
|
|
22
49
|
* with a review decision.
|
|
@@ -521,6 +548,10 @@ declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: Ga
|
|
|
521
548
|
* shallow copies (spread). Uses position + label so the
|
|
522
549
|
* key is identical for the original object and any
|
|
523
550
|
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
551
|
+
*
|
|
552
|
+
* @deprecated No longer used internally: coref alias
|
|
553
|
+
* links travel on the entities themselves
|
|
554
|
+
* (`corefSourceText`). Kept for API compatibility.
|
|
524
555
|
*/
|
|
525
556
|
declare const corefKey: (e: Entity) => string;
|
|
526
557
|
/**
|
|
@@ -582,17 +613,6 @@ type PipelineContext = {
|
|
|
582
613
|
zoneHeadingPatterns: RegExp[] | null;
|
|
583
614
|
zoneSigningPatterns: RegExp[] | null;
|
|
584
615
|
zoneInitPromise: Promise<void> | null;
|
|
585
|
-
/**
|
|
586
|
-
* Maps coreference entities to their source entity
|
|
587
|
-
* text. Populated by findCoreferenceSpans, consumed
|
|
588
|
-
* by buildPlaceholderMap for consistent placeholder
|
|
589
|
-
* numbering across aliases and source entities.
|
|
590
|
-
*
|
|
591
|
-
* Keyed by `start:end:label` composite string so
|
|
592
|
-
* lookups survive shallow copies (e.g. from
|
|
593
|
-
* mergeAndDedup's spread operator).
|
|
594
|
-
*/
|
|
595
|
-
corefSourceMap: Map<string, string>;
|
|
596
616
|
};
|
|
597
617
|
/** Create a fresh, empty pipeline context. */
|
|
598
618
|
declare const createPipelineContext: () => PipelineContext;
|
|
@@ -663,12 +683,11 @@ declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
|
|
|
663
683
|
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
664
684
|
* and N is a 1-based counter per label.
|
|
665
685
|
*
|
|
666
|
-
* @param
|
|
667
|
-
*
|
|
668
|
-
*
|
|
669
|
-
* Defaults to `defaultContext` for single-tenant usage.
|
|
686
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
687
|
+
* coref alias links now travel on the entities
|
|
688
|
+
* themselves (`corefSourceText`).
|
|
670
689
|
*/
|
|
671
|
-
declare const buildPlaceholderMap: (entities: Entity[],
|
|
690
|
+
declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext) => Map<string, string>;
|
|
672
691
|
/**
|
|
673
692
|
* Apply redactions to the source text, replacing each
|
|
674
693
|
* confirmed entity span using the configured operator.
|
|
@@ -811,11 +830,14 @@ declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?:
|
|
|
811
830
|
* character before the start and after the end are NOT
|
|
812
831
|
* word characters (letter/digit).
|
|
813
832
|
*
|
|
814
|
-
*
|
|
815
|
-
*
|
|
816
|
-
*
|
|
833
|
+
* Each returned alias carries `corefSourceText` linking
|
|
834
|
+
* it to its source entity text, for consistent
|
|
835
|
+
* placeholder numbering.
|
|
836
|
+
*
|
|
837
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
838
|
+
* alias links now travel on the entities themselves.
|
|
817
839
|
*/
|
|
818
|
-
declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[],
|
|
840
|
+
declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ctx?: PipelineContext) => Entity[];
|
|
819
841
|
//#endregion
|
|
820
842
|
//#region src/detectors/org-propagation.d.ts
|
|
821
843
|
/**
|
|
@@ -824,6 +846,12 @@ declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], ctx
|
|
|
824
846
|
* to get the base name, and re-scan the full text for
|
|
825
847
|
* bare mentions of that base name. Returns new entities
|
|
826
848
|
* for occurrences not already covered.
|
|
849
|
+
*
|
|
850
|
+
* Propagated mentions are coref aliases: each carries
|
|
851
|
+
* `corefSourceText` linking it to the full seed entity
|
|
852
|
+
* text, so placeholder numbering assigns the bare
|
|
853
|
+
* mention the same placeholder as its source ("Acme"
|
|
854
|
+
* and "Acme Corp." both become [ORGANIZATION_1]).
|
|
827
855
|
*/
|
|
828
856
|
declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
|
|
829
857
|
//#endregion
|
package/dist/wasm.mjs
CHANGED
|
@@ -33,6 +33,10 @@ const isLegalFormsEnabled = (config) => config.enableLegalForms !== false;
|
|
|
33
33
|
* shallow copies (spread). Uses position + label so the
|
|
34
34
|
* key is identical for the original object and any
|
|
35
35
|
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
36
|
+
*
|
|
37
|
+
* @deprecated No longer used internally: coref alias
|
|
38
|
+
* links travel on the entities themselves
|
|
39
|
+
* (`corefSourceText`). Kept for API compatibility.
|
|
36
40
|
*/
|
|
37
41
|
const corefKey = (e) => `${e.start}:${e.end}:${e.label}`;
|
|
38
42
|
/** Create a fresh, empty pipeline context. */
|
|
@@ -62,8 +66,7 @@ const createPipelineContext = () => ({
|
|
|
62
66
|
roleStopSetPromise: null,
|
|
63
67
|
zoneHeadingPatterns: null,
|
|
64
68
|
zoneSigningPatterns: null,
|
|
65
|
-
zoneInitPromise: null
|
|
66
|
-
corefSourceMap: /* @__PURE__ */ new Map()
|
|
69
|
+
zoneInitPromise: null
|
|
67
70
|
});
|
|
68
71
|
/**
|
|
69
72
|
* Module-level default context. Used when callers
|
|
@@ -1761,11 +1764,14 @@ const isWordChar = (ch) => {
|
|
|
1761
1764
|
* character before the start and after the end are NOT
|
|
1762
1765
|
* word characters (letter/digit).
|
|
1763
1766
|
*
|
|
1764
|
-
*
|
|
1765
|
-
*
|
|
1766
|
-
*
|
|
1767
|
+
* Each returned alias carries `corefSourceText` linking
|
|
1768
|
+
* it to its source entity text, for consistent
|
|
1769
|
+
* placeholder numbering.
|
|
1770
|
+
*
|
|
1771
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
1772
|
+
* alias links now travel on the entities themselves.
|
|
1767
1773
|
*/
|
|
1768
|
-
const findCoreferenceSpans = (fullText, terms,
|
|
1774
|
+
const findCoreferenceSpans = (fullText, terms, _ctx = defaultContext) => {
|
|
1769
1775
|
const results = [];
|
|
1770
1776
|
for (const term of terms) {
|
|
1771
1777
|
let searchFrom = 0;
|
|
@@ -1779,16 +1785,15 @@ const findCoreferenceSpans = (fullText, terms, ctx = defaultContext) => {
|
|
|
1779
1785
|
searchFrom = idx + 1;
|
|
1780
1786
|
continue;
|
|
1781
1787
|
}
|
|
1782
|
-
|
|
1788
|
+
results.push({
|
|
1783
1789
|
start: idx,
|
|
1784
1790
|
end: matchEnd,
|
|
1785
1791
|
label: term.label,
|
|
1786
1792
|
text: term.alias,
|
|
1787
1793
|
score: .95,
|
|
1788
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
results.push(entity);
|
|
1794
|
+
source: DETECTION_SOURCES.COREFERENCE,
|
|
1795
|
+
corefSourceText: term.sourceText
|
|
1796
|
+
});
|
|
1792
1797
|
searchFrom = matchEnd;
|
|
1793
1798
|
}
|
|
1794
1799
|
}
|
|
@@ -8055,32 +8060,216 @@ const initNameCorpus = (ctx = defaultContext, dictionaries, languages) => {
|
|
|
8055
8060
|
ctx.nameCorpusPromise = promise;
|
|
8056
8061
|
return promise;
|
|
8057
8062
|
};
|
|
8058
|
-
const
|
|
8059
|
-
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8063
|
+
const DECLENSION_RULES = [
|
|
8064
|
+
{
|
|
8065
|
+
ending: "",
|
|
8066
|
+
gate: /[bdfghklmnpqrstvwxzł]$/u,
|
|
8067
|
+
forms: [
|
|
8068
|
+
"a",
|
|
8069
|
+
"u",
|
|
8070
|
+
"ovi",
|
|
8071
|
+
"em",
|
|
8072
|
+
"om"
|
|
8073
|
+
]
|
|
8074
|
+
},
|
|
8075
|
+
{
|
|
8076
|
+
ending: "",
|
|
8077
|
+
gate: /[bdflmnpstvwz]$/u,
|
|
8078
|
+
forms: ["e"]
|
|
8079
|
+
},
|
|
8080
|
+
{
|
|
8081
|
+
ending: "",
|
|
8082
|
+
gate: /[cčďťňřšžjľ]$/u,
|
|
8083
|
+
forms: [
|
|
8084
|
+
"e",
|
|
8085
|
+
"i",
|
|
8086
|
+
"a",
|
|
8087
|
+
"ovi",
|
|
8088
|
+
"em",
|
|
8089
|
+
"om"
|
|
8090
|
+
]
|
|
8091
|
+
},
|
|
8092
|
+
{
|
|
8093
|
+
ending: "ek",
|
|
8094
|
+
gate: /[^aeiouyáéěíóôúůý]ek$/u,
|
|
8095
|
+
forms: [
|
|
8096
|
+
"ka",
|
|
8097
|
+
"ku",
|
|
8098
|
+
"kovi",
|
|
8099
|
+
"kem",
|
|
8100
|
+
"kom"
|
|
8101
|
+
]
|
|
8102
|
+
},
|
|
8103
|
+
{
|
|
8104
|
+
ending: "el",
|
|
8105
|
+
gate: /[^aeiouyáéěíóôúůý]el$/u,
|
|
8106
|
+
forms: [
|
|
8107
|
+
"la",
|
|
8108
|
+
"lu",
|
|
8109
|
+
"le",
|
|
8110
|
+
"lovi",
|
|
8111
|
+
"lem",
|
|
8112
|
+
"lom"
|
|
8113
|
+
]
|
|
8114
|
+
},
|
|
8115
|
+
{
|
|
8116
|
+
ending: "ec",
|
|
8117
|
+
gate: /[^aeiouyáéěíóôúůý]ec$/u,
|
|
8118
|
+
forms: [
|
|
8119
|
+
"ce",
|
|
8120
|
+
"ci",
|
|
8121
|
+
"covi",
|
|
8122
|
+
"cem",
|
|
8123
|
+
"com"
|
|
8124
|
+
]
|
|
8125
|
+
},
|
|
8126
|
+
{
|
|
8127
|
+
ending: "a",
|
|
8128
|
+
gate: /[^cčďťňřšžji]a$/u,
|
|
8129
|
+
forms: [
|
|
8130
|
+
"y",
|
|
8131
|
+
"u",
|
|
8132
|
+
"o",
|
|
8133
|
+
"ou",
|
|
8134
|
+
"ovi"
|
|
8135
|
+
]
|
|
8136
|
+
},
|
|
8137
|
+
{
|
|
8138
|
+
ending: "a",
|
|
8139
|
+
gate: /[cčďťňřšžj]a$/u,
|
|
8140
|
+
forms: [
|
|
8141
|
+
"i",
|
|
8142
|
+
"u",
|
|
8143
|
+
"o",
|
|
8144
|
+
"ou",
|
|
8145
|
+
"ovi"
|
|
8146
|
+
]
|
|
8147
|
+
},
|
|
8148
|
+
{
|
|
8149
|
+
ending: "a",
|
|
8150
|
+
gate: /ia$/u,
|
|
8151
|
+
forms: [
|
|
8152
|
+
"e",
|
|
8153
|
+
"i",
|
|
8154
|
+
"u",
|
|
8155
|
+
"ou"
|
|
8156
|
+
]
|
|
8157
|
+
},
|
|
8158
|
+
{
|
|
8159
|
+
ending: "ka",
|
|
8160
|
+
gate: /ka$/u,
|
|
8161
|
+
forms: ["ce"]
|
|
8162
|
+
},
|
|
8163
|
+
{
|
|
8164
|
+
ending: "ra",
|
|
8165
|
+
gate: /ra$/u,
|
|
8166
|
+
forms: ["ře", "re"]
|
|
8167
|
+
},
|
|
8168
|
+
{
|
|
8169
|
+
ending: "ha",
|
|
8170
|
+
gate: /ha$/u,
|
|
8171
|
+
forms: ["ze"]
|
|
8172
|
+
},
|
|
8173
|
+
{
|
|
8174
|
+
ending: "ga",
|
|
8175
|
+
gate: /ga$/u,
|
|
8176
|
+
forms: ["ze"]
|
|
8177
|
+
},
|
|
8178
|
+
{
|
|
8179
|
+
ending: "cha",
|
|
8180
|
+
gate: /cha$/u,
|
|
8181
|
+
forms: ["še"]
|
|
8182
|
+
},
|
|
8183
|
+
{
|
|
8184
|
+
ending: "a",
|
|
8185
|
+
gate: /[bdfmnptv]a$/u,
|
|
8186
|
+
forms: ["ě", "e"]
|
|
8187
|
+
},
|
|
8188
|
+
{
|
|
8189
|
+
ending: "a",
|
|
8190
|
+
gate: /[szl]a$/u,
|
|
8191
|
+
forms: ["e"]
|
|
8192
|
+
},
|
|
8193
|
+
{
|
|
8194
|
+
ending: "á",
|
|
8195
|
+
gate: /á$/u,
|
|
8196
|
+
forms: [
|
|
8197
|
+
"é",
|
|
8198
|
+
"ou",
|
|
8199
|
+
"ej",
|
|
8200
|
+
"ú"
|
|
8201
|
+
]
|
|
8202
|
+
},
|
|
8203
|
+
{
|
|
8204
|
+
ending: "ý",
|
|
8205
|
+
gate: /ý$/u,
|
|
8206
|
+
forms: [
|
|
8207
|
+
"ého",
|
|
8208
|
+
"ému",
|
|
8209
|
+
"ém",
|
|
8210
|
+
"ým"
|
|
8211
|
+
]
|
|
8212
|
+
},
|
|
8213
|
+
{
|
|
8214
|
+
ending: "",
|
|
8215
|
+
gate: /[íiy]$/u,
|
|
8216
|
+
forms: [
|
|
8217
|
+
"ho",
|
|
8218
|
+
"mu",
|
|
8219
|
+
"m"
|
|
8220
|
+
]
|
|
8221
|
+
},
|
|
8222
|
+
{
|
|
8223
|
+
ending: "e",
|
|
8224
|
+
gate: /[^aeouyáéěíóôúůý]e$/u,
|
|
8225
|
+
forms: ["i", "í"]
|
|
8226
|
+
},
|
|
8227
|
+
{
|
|
8228
|
+
ending: "o",
|
|
8229
|
+
gate: /o$/u,
|
|
8230
|
+
forms: [
|
|
8231
|
+
"a",
|
|
8232
|
+
"ovi",
|
|
8233
|
+
"em",
|
|
8234
|
+
"om"
|
|
8235
|
+
]
|
|
8236
|
+
}
|
|
8066
8237
|
];
|
|
8067
8238
|
/**
|
|
8068
|
-
*
|
|
8069
|
-
*
|
|
8070
|
-
*
|
|
8071
|
-
*
|
|
8072
|
-
*
|
|
8073
|
-
|
|
8074
|
-
|
|
8239
|
+
* Generate declined Czech/Slovak variants of a
|
|
8240
|
+
* nominative name. Only variants licensed by the
|
|
8241
|
+
* ending-shape rules are produced, which bounds the
|
|
8242
|
+
* deny-list AC pattern growth. Returns an empty array
|
|
8243
|
+
* for names too short to decline safely.
|
|
8244
|
+
*/
|
|
8245
|
+
const expandNameDeclensions = (name) => {
|
|
8246
|
+
if (name.length < 3) return [];
|
|
8247
|
+
const lower = name.toLowerCase();
|
|
8248
|
+
const variants = [];
|
|
8249
|
+
for (const rule of DECLENSION_RULES) {
|
|
8250
|
+
if (!rule.gate.test(lower)) continue;
|
|
8251
|
+
const stem = name.slice(0, name.length - rule.ending.length);
|
|
8252
|
+
if (stem.length < 2) continue;
|
|
8253
|
+
for (const form of rule.forms) variants.push(stem + form);
|
|
8254
|
+
}
|
|
8255
|
+
return variants;
|
|
8256
|
+
};
|
|
8257
|
+
/**
|
|
8258
|
+
* Strip Czech/Slovak case endings from a token using
|
|
8259
|
+
* the inverse of DECLENSION_RULES. Returns candidate
|
|
8260
|
+
* nominative forms; each candidate is validated against
|
|
8261
|
+
* the rule's shape gate, so only bases the paradigm
|
|
8262
|
+
* could actually have declined are proposed. Callers
|
|
8263
|
+
* check candidates against the name corpus.
|
|
8075
8264
|
*/
|
|
8076
8265
|
const stripInflection = (token) => {
|
|
8077
8266
|
const candidates = [];
|
|
8078
|
-
for (const
|
|
8079
|
-
|
|
8080
|
-
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
|
|
8267
|
+
for (const rule of DECLENSION_RULES) for (const form of rule.forms) {
|
|
8268
|
+
if (token.length <= form.length + 2 || !token.endsWith(form)) continue;
|
|
8269
|
+
const base = token.slice(0, token.length - form.length) + rule.ending;
|
|
8270
|
+
if (!/^\p{Lu}/u.test(base)) continue;
|
|
8271
|
+
if (!rule.gate.test(base.toLowerCase())) continue;
|
|
8272
|
+
candidates.push(base);
|
|
8084
8273
|
}
|
|
8085
8274
|
return candidates;
|
|
8086
8275
|
};
|
|
@@ -8340,7 +8529,7 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
|
|
|
8340
8529
|
const SLASH_S_RE = /\/s\/[ \t]*/g;
|
|
8341
8530
|
const LABELLED_NAME_RE = /\b(?:By|Name)[ \t]*:[ \t]*(?:\/s\/[ \t]+)?([^\n]*?)(?=\s{3,}|[\t]|\n|$)/gi;
|
|
8342
8531
|
const WITNESS_ANCHOR_RE = /\bIN WITNESS WHEREOF\b[^\n]*\n/gi;
|
|
8343
|
-
const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
|
|
8532
|
+
const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|da|das|do|dos|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
|
|
8344
8533
|
const CAP_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'\\-]{0,30}";
|
|
8345
8534
|
const NAME_SHAPE_RE = new RegExp(`^${CAP_TOKEN}(?:[ \\t]+(?:${NAME_PARTICLE}|${CAP_TOKEN})){1,4}$`, "u");
|
|
8346
8535
|
const MAX_NAME_LEN = 60;
|
|
@@ -10138,6 +10327,8 @@ const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
|
|
|
10138
10327
|
* etc.), skip the comma and degree, then continue.
|
|
10139
10328
|
*/
|
|
10140
10329
|
const POST_NOMINAL_RE = new RegExp(`^,\\s*(?:${POST_NOMINALS.toSorted((a, b) => b.length - a.length).map((d) => d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\./g, "\\.\\s*")).join("|")})\\.?`, "i");
|
|
10330
|
+
const PERSON_NAME_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'’\\-]*";
|
|
10331
|
+
const PERSON_NAME_RUN_RE = new RegExp(`^${PERSON_NAME_TOKEN}(?:,?[ \\t]+(?:${`(?:${NAME_PARTICLE}[ \\t]+|d['’])`}){0,2}${PERSON_NAME_TOKEN})*`, "u");
|
|
10141
10332
|
let cachedLegalFormCheckSource = null;
|
|
10142
10333
|
let cachedLegalFormCheckForms = [];
|
|
10143
10334
|
const LEGAL_FORM_ALNUM_RE = /[\p{L}\p{N}]/u;
|
|
@@ -10805,9 +10996,16 @@ const processTriggerMatches = (allMatches, sliceStart, sliceEnd, fullText, rules
|
|
|
10805
10996
|
if (!applyValidations(value.text, rule.validations)) continue;
|
|
10806
10997
|
if (rule.label === "phone number" && !isPlausiblePhoneTriggerValue(value.text)) continue;
|
|
10807
10998
|
const entityStart = rule.includeTrigger ? match.start : value.start;
|
|
10808
|
-
|
|
10809
|
-
|
|
10999
|
+
let entityEnd = value.end;
|
|
11000
|
+
let entityText = fullText.slice(entityStart, entityEnd);
|
|
10810
11001
|
const effectiveLabel = rule.label === "person" && hasKnownLegalFormSuffix(entityText) ? "organization" : rule.label;
|
|
11002
|
+
if (effectiveLabel === "person") {
|
|
11003
|
+
const run = PERSON_NAME_RUN_RE.exec(value.text);
|
|
11004
|
+
if (run !== null && run[0].length < value.text.length) {
|
|
11005
|
+
entityEnd = value.start + run[0].length;
|
|
11006
|
+
entityText = fullText.slice(entityStart, entityEnd);
|
|
11007
|
+
}
|
|
11008
|
+
}
|
|
10811
11009
|
results.push({
|
|
10812
11010
|
start: entityStart,
|
|
10813
11011
|
end: entityEnd,
|
|
@@ -11431,6 +11629,9 @@ const EMPTY_ALLOW_LIST = /* @__PURE__ */ new Set();
|
|
|
11431
11629
|
const getAllowList = (ctx) => ctx.allowList ?? EMPTY_ALLOW_LIST;
|
|
11432
11630
|
let commonWordsPromise = null;
|
|
11433
11631
|
let commonWordsCache = null;
|
|
11632
|
+
const EMPTY_COMMON_WORDS = /* @__PURE__ */ new Set();
|
|
11633
|
+
/** Sync accessor — returns empty set before init. */
|
|
11634
|
+
const getCommonWords = () => commonWordsCache ?? EMPTY_COMMON_WORDS;
|
|
11434
11635
|
const loadCommonWords = () => {
|
|
11435
11636
|
if (commonWordsCache) return Promise.resolve(commonWordsCache);
|
|
11436
11637
|
if (commonWordsPromise) return commonWordsPromise;
|
|
@@ -11878,8 +12079,21 @@ const appendNameCorpusEntries = (config, ctx, patternList, labelList, sourceList
|
|
|
11878
12079
|
sourceList.push(source);
|
|
11879
12080
|
}
|
|
11880
12081
|
};
|
|
11881
|
-
|
|
11882
|
-
|
|
12082
|
+
const commonWords = getCommonWords();
|
|
12083
|
+
const addDeclinedVariants = (name, source) => {
|
|
12084
|
+
for (const variant of expandNameDeclensions(name)) {
|
|
12085
|
+
if (commonWords.has(variant.toLowerCase())) continue;
|
|
12086
|
+
addNameEntry(variant, source);
|
|
12087
|
+
}
|
|
12088
|
+
};
|
|
12089
|
+
for (const name of getNameCorpusFirstNames(ctx)) {
|
|
12090
|
+
addNameEntry(name, "first-name");
|
|
12091
|
+
addDeclinedVariants(name, "first-name");
|
|
12092
|
+
}
|
|
12093
|
+
for (const name of getNameCorpusSurnames(ctx)) {
|
|
12094
|
+
addNameEntry(name, "surname");
|
|
12095
|
+
addDeclinedVariants(name, "surname");
|
|
12096
|
+
}
|
|
11883
12097
|
for (const title of getNameCorpusTitles(ctx)) {
|
|
11884
12098
|
const norm = stripCuratedPatternSyntax(normalizeForSearch(title));
|
|
11885
12099
|
if (norm.length === 0) continue;
|
|
@@ -12656,25 +12870,33 @@ const isCallerOwnedEntity$2 = (entity) => entity.sourceDetail === "custom-deny-l
|
|
|
12656
12870
|
* to get the base name, and re-scan the full text for
|
|
12657
12871
|
* bare mentions of that base name. Returns new entities
|
|
12658
12872
|
* for occurrences not already covered.
|
|
12873
|
+
*
|
|
12874
|
+
* Propagated mentions are coref aliases: each carries
|
|
12875
|
+
* `corefSourceText` linking it to the full seed entity
|
|
12876
|
+
* text, so placeholder numbering assigns the bare
|
|
12877
|
+
* mention the same placeholder as its source ("Acme"
|
|
12878
|
+
* and "Acme Corp." both become [ORGANIZATION_1]).
|
|
12659
12879
|
*/
|
|
12660
12880
|
const propagateOrgNames = (entities, fullText) => {
|
|
12661
|
-
const
|
|
12662
|
-
const seenBases = /* @__PURE__ */ new Set();
|
|
12881
|
+
const seedByBase = /* @__PURE__ */ new Map();
|
|
12663
12882
|
for (const e of entities) {
|
|
12664
12883
|
if (e.label !== "organization") continue;
|
|
12665
12884
|
if (isCallerOwnedEntity$2(e)) continue;
|
|
12666
12885
|
for (const suffix of LEGAL_SUFFIXES) if (e.text.endsWith(suffix)) {
|
|
12667
12886
|
const base = e.text.slice(0, -suffix.length).replace(TRAILING_SEP, "").trim();
|
|
12668
|
-
if (base.length >= 3
|
|
12669
|
-
|
|
12670
|
-
|
|
12887
|
+
if (base.length >= 3) {
|
|
12888
|
+
const existing = seedByBase.get(base);
|
|
12889
|
+
if (existing === void 0) seedByBase.set(base, {
|
|
12671
12890
|
baseName: base,
|
|
12672
|
-
label: e.label
|
|
12891
|
+
label: e.label,
|
|
12892
|
+
sourceText: e.text
|
|
12673
12893
|
});
|
|
12894
|
+
else if (existing.sourceText !== e.text) existing.sourceText = base;
|
|
12674
12895
|
}
|
|
12675
12896
|
break;
|
|
12676
12897
|
}
|
|
12677
12898
|
}
|
|
12899
|
+
const seeds = [...seedByBase.values()];
|
|
12678
12900
|
if (seeds.length === 0) return [];
|
|
12679
12901
|
const covered = entities.map((e) => [e.start, e.end]);
|
|
12680
12902
|
const isOverlapping = (start, end) => covered.some(([cs, ce]) => start < ce && end > cs);
|
|
@@ -12708,7 +12930,8 @@ const propagateOrgNames = (entities, fullText) => {
|
|
|
12708
12930
|
label,
|
|
12709
12931
|
text: fullText.slice(spanStart, matchEnd),
|
|
12710
12932
|
score: ORG_PROPAGATION_SCORE,
|
|
12711
|
-
source: DETECTION_SOURCES.COREFERENCE
|
|
12933
|
+
source: DETECTION_SOURCES.COREFERENCE,
|
|
12934
|
+
corefSourceText: seed.sourceText
|
|
12712
12935
|
});
|
|
12713
12936
|
covered.push([spanStart, matchEnd]);
|
|
12714
12937
|
}
|
|
@@ -14797,7 +15020,6 @@ const runPipeline = async (options) => {
|
|
|
14797
15020
|
const merged = filterFalsePositives(postOrgEntities, ctx, fullText);
|
|
14798
15021
|
if (merged.length < postOrgEntities.length) log("filter", `removed ${postOrgEntities.length - merged.length} FPs`);
|
|
14799
15022
|
checkAbort(signal);
|
|
14800
|
-
ctx.corefSourceMap.clear();
|
|
14801
15023
|
if (config.enableCoreference) {
|
|
14802
15024
|
if (!legalFormsEnabled) await warmLegalRoleHeads();
|
|
14803
15025
|
const terms = await extractDefinedTerms(fullText, merged.filter((entity) => !isCallerOwnedEntity(entity)), ctx);
|
|
@@ -14863,12 +15085,11 @@ const normalizeEntityText = (label, text) => {
|
|
|
14863
15085
|
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
14864
15086
|
* and N is a 1-based counter per label.
|
|
14865
15087
|
*
|
|
14866
|
-
* @param
|
|
14867
|
-
*
|
|
14868
|
-
*
|
|
14869
|
-
* Defaults to `defaultContext` for single-tenant usage.
|
|
15088
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
15089
|
+
* coref alias links now travel on the entities
|
|
15090
|
+
* themselves (`corefSourceText`).
|
|
14870
15091
|
*/
|
|
14871
|
-
const buildPlaceholderMap = (entities,
|
|
15092
|
+
const buildPlaceholderMap = (entities, _ctx = defaultContext) => {
|
|
14872
15093
|
const counters = /* @__PURE__ */ new Map();
|
|
14873
15094
|
const textLabelToPlaceholder = /* @__PURE__ */ new Map();
|
|
14874
15095
|
const normalizedToPlaceholder = /* @__PURE__ */ new Map();
|
|
@@ -14877,9 +15098,9 @@ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
|
14877
15098
|
const compositeKey = `${entity.label}\0${entity.text}`;
|
|
14878
15099
|
if (textLabelToPlaceholder.has(compositeKey)) continue;
|
|
14879
15100
|
const labelKey = entity.label.toUpperCase().replace(WHITESPACE_RE, "_");
|
|
14880
|
-
const sourceText =
|
|
14881
|
-
|
|
14882
|
-
|
|
15101
|
+
const sourceText = entity.source === "coreference" ? entity.corefSourceText : void 0;
|
|
15102
|
+
const sourceNormalizedKey = sourceText === void 0 ? void 0 : `${labelKey}\0${normalizeEntityText(entity.label, sourceText)}`;
|
|
15103
|
+
if (sourceNormalizedKey !== void 0) {
|
|
14883
15104
|
const sourceExisting = normalizedToPlaceholder.get(sourceNormalizedKey);
|
|
14884
15105
|
if (sourceExisting) {
|
|
14885
15106
|
textLabelToPlaceholder.set(compositeKey, sourceExisting);
|
|
@@ -14890,6 +15111,7 @@ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
|
14890
15111
|
const existing = normalizedToPlaceholder.get(normalizedKey);
|
|
14891
15112
|
if (existing) {
|
|
14892
15113
|
textLabelToPlaceholder.set(compositeKey, existing);
|
|
15114
|
+
if (sourceNormalizedKey !== void 0) normalizedToPlaceholder.set(sourceNormalizedKey, existing);
|
|
14893
15115
|
continue;
|
|
14894
15116
|
}
|
|
14895
15117
|
const count = (counters.get(labelKey) ?? 0) + 1;
|
|
@@ -14897,6 +15119,7 @@ const buildPlaceholderMap = (entities, ctx = defaultContext) => {
|
|
|
14897
15119
|
const placeholder = `[${labelKey}_${count}]`;
|
|
14898
15120
|
textLabelToPlaceholder.set(compositeKey, placeholder);
|
|
14899
15121
|
normalizedToPlaceholder.set(normalizedKey, placeholder);
|
|
15122
|
+
if (sourceNormalizedKey !== void 0) normalizedToPlaceholder.set(sourceNormalizedKey, placeholder);
|
|
14900
15123
|
}
|
|
14901
15124
|
return textLabelToPlaceholder;
|
|
14902
15125
|
};
|
|
@@ -14939,7 +15162,7 @@ const redactText = (fullText, entities, config = DEFAULT_OPERATOR_CONFIG, ctx =
|
|
|
14939
15162
|
const replacement = operator.apply(entity.text, entity.label, placeholder, config.redactString);
|
|
14940
15163
|
parts.push(replacement);
|
|
14941
15164
|
operatorMap.set(placeholder, opType);
|
|
14942
|
-
if (operator.reversibility === "reversible" && !redactionMap.has(placeholder)) redactionMap.set(placeholder, entity.text);
|
|
15165
|
+
if (operator.reversibility === "reversible" && !redactionMap.has(placeholder)) redactionMap.set(placeholder, entity.source === "coreference" ? entity.corefSourceText : entity.text);
|
|
14943
15166
|
cursor = entity.end;
|
|
14944
15167
|
}
|
|
14945
15168
|
if (cursor < fullText.length) parts.push(fullText.slice(cursor));
|