@stll/anonymize-wasm 1.4.7 → 1.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/triggers.global.mjs +73 -28
- package/dist/wasm.d.mts +53 -25
- package/dist/wasm.mjs +314 -58
- package/dist/wasm.mjs.map +1 -1
- package/package.json +3 -3
package/dist/triggers.global.mjs
CHANGED
|
@@ -1,33 +1,78 @@
|
|
|
1
1
|
//#region src/data/triggers.global.json
|
|
2
|
-
var triggers_global_default = [
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
"
|
|
8
|
-
|
|
2
|
+
var triggers_global_default = [
|
|
3
|
+
{
|
|
4
|
+
"id": "swift-bic",
|
|
5
|
+
"triggers": ["SWIFT", "BIC"],
|
|
6
|
+
"label": "bank account number",
|
|
7
|
+
"strategy": {
|
|
8
|
+
"type": "n-words",
|
|
9
|
+
"count": 1
|
|
10
|
+
},
|
|
11
|
+
"extensions": ["add-colon"],
|
|
12
|
+
"validations": [{
|
|
13
|
+
"type": "matches-pattern",
|
|
14
|
+
"pattern": "^[A-Za-z]{4}[A-Za-z]{2}[A-Za-z0-9]{2}(?:[A-Za-z0-9]{3})?$"
|
|
15
|
+
}]
|
|
9
16
|
},
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
"
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
17
|
+
{
|
|
18
|
+
"id": "us-aba-routing",
|
|
19
|
+
"triggers": [
|
|
20
|
+
"ABA",
|
|
21
|
+
"ABA Number",
|
|
22
|
+
"RTN",
|
|
23
|
+
"Routing Number",
|
|
24
|
+
"Routing No.",
|
|
25
|
+
"Routing Transit Number"
|
|
26
|
+
],
|
|
27
|
+
"label": "bank account number",
|
|
28
|
+
"strategy": {
|
|
29
|
+
"type": "n-words",
|
|
30
|
+
"count": 1
|
|
31
|
+
},
|
|
32
|
+
"extensions": ["add-colon"],
|
|
33
|
+
"validations": [{ "type": "has-digits" }, {
|
|
34
|
+
"type": "valid-id",
|
|
35
|
+
"validator": "us.rtn"
|
|
36
|
+
}]
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "intl-labeled-phone",
|
|
40
|
+
"triggers": [
|
|
41
|
+
"fax",
|
|
42
|
+
"telecopy",
|
|
43
|
+
"telecopier",
|
|
44
|
+
"facsimile",
|
|
45
|
+
"telephone",
|
|
46
|
+
"phone number",
|
|
47
|
+
"contact phone number"
|
|
48
|
+
],
|
|
49
|
+
"label": "phone number",
|
|
50
|
+
"strategy": {
|
|
51
|
+
"type": "match-pattern",
|
|
52
|
+
"pattern": "(?:\\+\\d{1,3}[\\s.\\-]?)?\\(?\\d{1,4}\\)?(?:[\\s.\\-]\\d{1,4}){2,4}"
|
|
53
|
+
},
|
|
54
|
+
"validations": [{
|
|
55
|
+
"type": "matches-pattern",
|
|
56
|
+
"pattern": "(?:\\d[^\\d]*){6,}"
|
|
57
|
+
}]
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "amount-prefix",
|
|
61
|
+
"triggers": [
|
|
62
|
+
"ve výši",
|
|
63
|
+
"in the amount of",
|
|
64
|
+
"in Höhe von",
|
|
65
|
+
"d'un montant de",
|
|
66
|
+
"w wysokości",
|
|
67
|
+
"nell'importo di",
|
|
68
|
+
"por un importe de"
|
|
69
|
+
],
|
|
70
|
+
"label": "monetary amount",
|
|
71
|
+
"strategy": { "type": "to-next-comma" },
|
|
72
|
+
"extensions": ["add-colon"],
|
|
73
|
+
"validations": [{ "type": "has-digits" }]
|
|
74
|
+
}
|
|
75
|
+
];
|
|
31
76
|
//#endregion
|
|
32
77
|
export { triggers_global_default as default };
|
|
33
78
|
|
package/dist/wasm.d.mts
CHANGED
|
@@ -5,18 +5,45 @@ import { Tokenizer } from "@huggingface/tokenizers";
|
|
|
5
5
|
|
|
6
6
|
//#region src/types.d.ts
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
* Every detection layer produces these.
|
|
8
|
+
* Fields shared by every entity span in the source text.
|
|
10
9
|
*/
|
|
11
|
-
type
|
|
10
|
+
type EntityBase = {
|
|
12
11
|
start: number;
|
|
13
12
|
end: number;
|
|
14
13
|
label: string;
|
|
15
14
|
text: string;
|
|
16
15
|
score: number;
|
|
17
|
-
source: DetectionSource;
|
|
18
16
|
sourceDetail?: "custom-deny-list" | "custom-regex" | "gazetteer-extension";
|
|
19
17
|
};
|
|
18
|
+
/**
|
|
19
|
+
* A PII entity span found by a primary detection layer
|
|
20
|
+
* (regex, NER, legal forms, deny list, ...).
|
|
21
|
+
*/
|
|
22
|
+
type DetectedEntity = EntityBase & {
|
|
23
|
+
source: Exclude<DetectionSource, typeof DETECTION_SOURCES.COREFERENCE>;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* An alias mention of a previously detected entity: a
|
|
27
|
+
* defined term ("the Seller") or a propagated bare
|
|
28
|
+
* mention ("Acme" after "Acme Corp.").
|
|
29
|
+
*
|
|
30
|
+
* `corefSourceText` is required by construction, so an
|
|
31
|
+
* alias cannot exist without the link back to its source
|
|
32
|
+
* entity. Placeholder numbering reads it to give the
|
|
33
|
+
* alias the same placeholder as the source. The link
|
|
34
|
+
* travels with the entity instead of living in a
|
|
35
|
+
* side-channel map that a producer could forget to
|
|
36
|
+
* write — or that a later pass could clear.
|
|
37
|
+
*/
|
|
38
|
+
type CorefAliasEntity = EntityBase & {
|
|
39
|
+
source: typeof DETECTION_SOURCES.COREFERENCE; /** Full text of the source entity this alias refers to. */
|
|
40
|
+
corefSourceText: string;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* A detected PII entity span in the source text.
|
|
44
|
+
* Every detection layer produces these.
|
|
45
|
+
*/
|
|
46
|
+
type Entity = DetectedEntity | CorefAliasEntity;
|
|
20
47
|
/**
|
|
21
48
|
* Entity after human review. Extends the base Entity
|
|
22
49
|
* with a review decision.
|
|
@@ -117,7 +144,7 @@ type TriggerValidation = {
|
|
|
117
144
|
};
|
|
118
145
|
/** Built-in stdnum validators that can be referenced
|
|
119
146
|
* by `valid-id` validations. */
|
|
120
|
-
type ValidIdValidator = "br.cpf" | "br.cnpj";
|
|
147
|
+
type ValidIdValidator = "br.cpf" | "br.cnpj" | "us.rtn";
|
|
121
148
|
/** Auto-generated trigger variants — closed set. */
|
|
122
149
|
type TriggerExtension = "add-colon" | "add-trailing-space" | "add-colon-space" | "normalize-spaces";
|
|
123
150
|
/** V2 trigger config entry (JSON shape). */
|
|
@@ -521,6 +548,10 @@ declare const buildUnifiedSearch: (config: PipelineConfig, gazetteerEntries?: Ga
|
|
|
521
548
|
* shallow copies (spread). Uses position + label so the
|
|
522
549
|
* key is identical for the original object and any
|
|
523
550
|
* `{ ...entity }` copy produced by mergeAndDedup.
|
|
551
|
+
*
|
|
552
|
+
* @deprecated No longer used internally: coref alias
|
|
553
|
+
* links travel on the entities themselves
|
|
554
|
+
* (`corefSourceText`). Kept for API compatibility.
|
|
524
555
|
*/
|
|
525
556
|
declare const corefKey: (e: Entity) => string;
|
|
526
557
|
/**
|
|
@@ -582,17 +613,6 @@ type PipelineContext = {
|
|
|
582
613
|
zoneHeadingPatterns: RegExp[] | null;
|
|
583
614
|
zoneSigningPatterns: RegExp[] | null;
|
|
584
615
|
zoneInitPromise: Promise<void> | null;
|
|
585
|
-
/**
|
|
586
|
-
* Maps coreference entities to their source entity
|
|
587
|
-
* text. Populated by findCoreferenceSpans, consumed
|
|
588
|
-
* by buildPlaceholderMap for consistent placeholder
|
|
589
|
-
* numbering across aliases and source entities.
|
|
590
|
-
*
|
|
591
|
-
* Keyed by `start:end:label` composite string so
|
|
592
|
-
* lookups survive shallow copies (e.g. from
|
|
593
|
-
* mergeAndDedup's spread operator).
|
|
594
|
-
*/
|
|
595
|
-
corefSourceMap: Map<string, string>;
|
|
596
616
|
};
|
|
597
617
|
/** Create a fresh, empty pipeline context. */
|
|
598
618
|
declare const createPipelineContext: () => PipelineContext;
|
|
@@ -663,12 +683,11 @@ declare const runPipeline: (options: PipelineOptions) => Promise<Entity[]>;
|
|
|
663
683
|
* Placeholder format: [LABEL_N] where LABEL is uppercase
|
|
664
684
|
* and N is a 1-based counter per label.
|
|
665
685
|
*
|
|
666
|
-
* @param
|
|
667
|
-
*
|
|
668
|
-
*
|
|
669
|
-
* Defaults to `defaultContext` for single-tenant usage.
|
|
686
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
687
|
+
* coref alias links now travel on the entities
|
|
688
|
+
* themselves (`corefSourceText`).
|
|
670
689
|
*/
|
|
671
|
-
declare const buildPlaceholderMap: (entities: Entity[],
|
|
690
|
+
declare const buildPlaceholderMap: (entities: Entity[], _ctx?: PipelineContext) => Map<string, string>;
|
|
672
691
|
/**
|
|
673
692
|
* Apply redactions to the source text, replacing each
|
|
674
693
|
* confirmed entity span using the configured operator.
|
|
@@ -811,11 +830,14 @@ declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?:
|
|
|
811
830
|
* character before the start and after the end are NOT
|
|
812
831
|
* word characters (letter/digit).
|
|
813
832
|
*
|
|
814
|
-
*
|
|
815
|
-
*
|
|
816
|
-
*
|
|
833
|
+
* Each returned alias carries `corefSourceText` linking
|
|
834
|
+
* it to its source entity text, for consistent
|
|
835
|
+
* placeholder numbering.
|
|
836
|
+
*
|
|
837
|
+
* @param _ctx Unused. Kept for signature compatibility;
|
|
838
|
+
* alias links now travel on the entities themselves.
|
|
817
839
|
*/
|
|
818
|
-
declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[],
|
|
840
|
+
declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], _ctx?: PipelineContext) => Entity[];
|
|
819
841
|
//#endregion
|
|
820
842
|
//#region src/detectors/org-propagation.d.ts
|
|
821
843
|
/**
|
|
@@ -824,6 +846,12 @@ declare const findCoreferenceSpans: (fullText: string, terms: DefinedTerm[], ctx
|
|
|
824
846
|
* to get the base name, and re-scan the full text for
|
|
825
847
|
* bare mentions of that base name. Returns new entities
|
|
826
848
|
* for occurrences not already covered.
|
|
849
|
+
*
|
|
850
|
+
* Propagated mentions are coref aliases: each carries
|
|
851
|
+
* `corefSourceText` linking it to the full seed entity
|
|
852
|
+
* text, so placeholder numbering assigns the bare
|
|
853
|
+
* mention the same placeholder as its source ("Acme"
|
|
854
|
+
* and "Acme Corp." both become [ORGANIZATION_1]).
|
|
827
855
|
*/
|
|
828
856
|
declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entity[];
|
|
829
857
|
//#endregion
|