@elanlanguages/bridge-anonymization 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -1
- package/dist/crypto/pii-map-crypto.d.ts.map +1 -1
- package/dist/crypto/pii-map-crypto.js +8 -8
- package/dist/crypto/pii-map-crypto.js.map +1 -1
- package/dist/index.d.ts +25 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +103 -52
- package/dist/index.js.map +1 -1
- package/dist/ner/model-manager.d.ts.map +1 -1
- package/dist/ner/model-manager.js +10 -8
- package/dist/ner/model-manager.js.map +1 -1
- package/dist/ner/ner-model.d.ts.map +1 -1
- package/dist/ner/ner-model.js +10 -10
- package/dist/ner/ner-model.js.map +1 -1
- package/dist/ner/onnx-runtime.d.ts +3 -3
- package/dist/ner/onnx-runtime.d.ts.map +1 -1
- package/dist/ner/onnx-runtime.js +1 -1
- package/dist/ner/onnx-runtime.js.map +1 -1
- package/dist/ner/tokenizer.d.ts +26 -53
- package/dist/ner/tokenizer.d.ts.map +1 -1
- package/dist/ner/tokenizer.js +174 -196
- package/dist/ner/tokenizer.js.map +1 -1
- package/dist/pipeline/index.d.ts +7 -4
- package/dist/pipeline/index.d.ts.map +1 -1
- package/dist/pipeline/index.js +7 -4
- package/dist/pipeline/index.js.map +1 -1
- package/dist/pipeline/resolver.d.ts.map +1 -1
- package/dist/pipeline/resolver.js +3 -2
- package/dist/pipeline/resolver.js.map +1 -1
- package/dist/pipeline/semantic-data-loader.d.ts +157 -0
- package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
- package/dist/pipeline/semantic-data-loader.js +662 -0
- package/dist/pipeline/semantic-data-loader.js.map +1 -0
- package/dist/pipeline/semantic-enricher.d.ts +102 -0
- package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
- package/dist/pipeline/semantic-enricher.js +268 -0
- package/dist/pipeline/semantic-enricher.js.map +1 -0
- package/dist/pipeline/tagger.d.ts +52 -12
- package/dist/pipeline/tagger.d.ts.map +1 -1
- package/dist/pipeline/tagger.js +226 -21
- package/dist/pipeline/tagger.js.map +1 -1
- package/dist/pipeline/title-extractor.d.ts +79 -0
- package/dist/pipeline/title-extractor.d.ts.map +1 -0
- package/dist/pipeline/title-extractor.js +801 -0
- package/dist/pipeline/title-extractor.js.map +1 -0
- package/dist/types/index.d.ts +66 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +14 -3
- package/dist/types/index.js.map +1 -1
- package/dist/utils/index.d.ts +3 -3
- package/dist/utils/index.js +3 -3
- package/package.json +7 -5
package/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Bridge Anonymization
|
|
2
2
|
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+
[](https://codecov.io/github/elanlanguages/bridge-anonymization)
|
|
6
|
+
|
|
3
7
|
On-device PII anonymization module for high-privacy translation workflows. Detects and replaces Personally Identifiable Information (PII) with placeholder tags while maintaining an encrypted mapping for later rehydration.
|
|
4
8
|
|
|
5
9
|
## Features
|
|
@@ -70,6 +74,74 @@ const result = await anonymizeWithNER(
|
|
|
70
74
|
);
|
|
71
75
|
```
|
|
72
76
|
|
|
77
|
+
## Translation Workflow (Anonymize → Translate → Rehydrate)
|
|
78
|
+
|
|
79
|
+
The full workflow for privacy-preserving translation:
|
|
80
|
+
|
|
81
|
+
```typescript
|
|
82
|
+
import {
|
|
83
|
+
createAnonymizer,
|
|
84
|
+
decryptPIIMap,
|
|
85
|
+
rehydrate,
|
|
86
|
+
InMemoryKeyProvider
|
|
87
|
+
} from '@elanlanguages/bridge-anonymization';
|
|
88
|
+
|
|
89
|
+
// 1. Create a key provider (so you can decrypt later)
|
|
90
|
+
const keyProvider = new InMemoryKeyProvider();
|
|
91
|
+
|
|
92
|
+
// 2. Create anonymizer with key provider
|
|
93
|
+
const anonymizer = createAnonymizer({
|
|
94
|
+
ner: { mode: 'quantized' },
|
|
95
|
+
keyProvider: keyProvider // Important: keep reference to decrypt later!
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
await anonymizer.initialize();
|
|
99
|
+
|
|
100
|
+
// 3. Anonymize before translation
|
|
101
|
+
const original = 'Hello John Smith from Acme Corp in Berlin!';
|
|
102
|
+
const result = await anonymizer.anonymize(original);
|
|
103
|
+
|
|
104
|
+
console.log(result.anonymizedText);
|
|
105
|
+
// "Hello <PII type="PERSON" id="1"/> from <PII type="ORG" id="2"/> in <PII type="LOCATION" id="3"/>!"
|
|
106
|
+
|
|
107
|
+
// 4. Translate (the placeholders are preserved by translation services)
|
|
108
|
+
const translated = await bridgeTranslate(result.anonymizedText, { from: 'en', to: 'de' });
|
|
109
|
+
// "Hallo <PII type="PERSON" id="1"/> von <PII type="ORG" id="2"/> in <PII type="LOCATION" id="3"/>!"
|
|
110
|
+
|
|
111
|
+
// 5. Decrypt the PII map using the same key
|
|
112
|
+
const encryptionKey = await keyProvider.getKey();
|
|
113
|
+
const piiMap = decryptPIIMap(result.piiMap, encryptionKey);
|
|
114
|
+
|
|
115
|
+
// 6. Rehydrate - replace placeholders with original values
|
|
116
|
+
const rehydrated = rehydrate(translated, piiMap);
|
|
117
|
+
|
|
118
|
+
console.log(rehydrated);
|
|
119
|
+
// "Hallo John Smith von Acme Corp in Berlin!"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Key Points
|
|
123
|
+
|
|
124
|
+
- **Save the encryption key** - You need the same key to decrypt the PII map
|
|
125
|
+
- **Placeholders are XML-like** - Most translation services preserve them automatically
|
|
126
|
+
- **PII stays local** - Original values never leave your system during translation
|
|
127
|
+
|
|
128
|
+
### Production Key Management
|
|
129
|
+
|
|
130
|
+
For production, use a proper key provider:
|
|
131
|
+
|
|
132
|
+
```typescript
|
|
133
|
+
import { EnvKeyProvider } from '@elanlanguages/bridge-anonymization';
|
|
134
|
+
|
|
135
|
+
// Generate and store key: openssl rand -base64 32
|
|
136
|
+
// Set environment variable: export PII_ENCRYPTION_KEY=<base64-key>
|
|
137
|
+
|
|
138
|
+
const keyProvider = new EnvKeyProvider('PII_ENCRYPTION_KEY');
|
|
139
|
+
const anonymizer = createAnonymizer({
|
|
140
|
+
ner: { mode: 'quantized' },
|
|
141
|
+
keyProvider
|
|
142
|
+
});
|
|
143
|
+
```
|
|
144
|
+
|
|
73
145
|
## API Reference
|
|
74
146
|
|
|
75
147
|
### Configuration Options
|
|
@@ -255,7 +327,7 @@ anonymizer.getRegistry().register(customRecognizer);
|
|
|
255
327
|
|
|
256
328
|
## Model Management
|
|
257
329
|
|
|
258
|
-
Models are hosted on [Hugging Face Hub](https://huggingface.co/
|
|
330
|
+
Models are hosted on [Hugging Face Hub](https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx) and automatically downloaded on first use.
|
|
259
331
|
|
|
260
332
|
**Cache locations:**
|
|
261
333
|
- **macOS**: `~/Library/Caches/bridge-anonymization/models/`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pii-map-crypto.d.ts","sourceRoot":"","sources":["../../src/crypto/pii-map-crypto.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,EAAE,gBAIvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,GAAE,OAAO,CAAC,aAAa,CAAM,GAAG,MAAM,CAGxE;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,UAAU,GAAE,MAAe,GAC1B,MAAM,CAER;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,MAAM,GAAE,MAAW,GAAG,MAAM,CAExD;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,SAAS,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACrC,eAAe,CAwCjB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,eAAe,EAC1B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACrC,SAAS,CAuCX;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,sCAAsC;IACtC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1B,sCAAsC;IACtC,SAAS,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IACrD,OAAO,CAAC,GAAG,CAAS;gBAER,GAAG,CAAC,EAAE,MAAM;
|
|
1
|
+
{"version":3,"file":"pii-map-crypto.d.ts","sourceRoot":"","sources":["../../src/crypto/pii-map-crypto.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,EAAE,gBAIvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,GAAE,OAAO,CAAC,aAAa,CAAM,GAAG,MAAM,CAGxE;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,UAAU,GAAE,MAAe,GAC1B,MAAM,CAER;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,MAAM,GAAE,MAAW,GAAG,MAAM,CAExD;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,SAAS,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACrC,eAAe,CAwCjB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,eAAe,EAC1B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACrC,SAAS,CAuCX;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,sCAAsC;IACtC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1B,sCAAsC;IACtC,SAAS,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IACrD,OAAO,CAAC,GAAG,CAAS;gBAER,GAAG,CAAC,EAAE,MAAM;IAIxB,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;IAIzB,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;CAI7B;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,WAAW;IAChD,OAAO,CAAC,UAAU,CAAS;gBAEf,UAAU,GAAE,MAA6B;IAIrD,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;CAa1B;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAEhD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAK3D"}
|
|
@@ -116,12 +116,12 @@ export class InMemoryKeyProvider {
|
|
|
116
116
|
constructor(key) {
|
|
117
117
|
this.key = key ?? generateKey();
|
|
118
118
|
}
|
|
119
|
-
|
|
120
|
-
return this.key;
|
|
119
|
+
getKey() {
|
|
120
|
+
return Promise.resolve(this.key);
|
|
121
121
|
}
|
|
122
|
-
|
|
122
|
+
rotateKey() {
|
|
123
123
|
this.key = generateKey();
|
|
124
|
-
return this.key;
|
|
124
|
+
return Promise.resolve(this.key);
|
|
125
125
|
}
|
|
126
126
|
}
|
|
127
127
|
/**
|
|
@@ -133,16 +133,16 @@ export class EnvKeyProvider {
|
|
|
133
133
|
constructor(envVarName = 'PII_ENCRYPTION_KEY') {
|
|
134
134
|
this.envVarName = envVarName;
|
|
135
135
|
}
|
|
136
|
-
|
|
136
|
+
getKey() {
|
|
137
137
|
const keyBase64 = process.env[this.envVarName];
|
|
138
138
|
if (keyBase64 === undefined || keyBase64.length === 0) {
|
|
139
|
-
|
|
139
|
+
return Promise.reject(new Error(`Encryption key not found in environment variable: ${this.envVarName}`));
|
|
140
140
|
}
|
|
141
141
|
const key = Buffer.from(keyBase64, 'base64');
|
|
142
142
|
if (key.length !== 32) {
|
|
143
|
-
|
|
143
|
+
return Promise.reject(new Error(`Invalid key length from ${this.envVarName}: expected 32 bytes`));
|
|
144
144
|
}
|
|
145
|
-
return key;
|
|
145
|
+
return Promise.resolve(key);
|
|
146
146
|
}
|
|
147
147
|
}
|
|
148
148
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pii-map-crypto.js","sourceRoot":"","sources":["../../src/crypto/pii-map-crypto.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAgBjC;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD,SAAS,EAAE,aAAa;IACxB,QAAQ,EAAE,EAAE;IACZ,aAAa,EAAE,EAAE;CAClB,CAAC;AAUF;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,UAAkC,EAAE;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;IACpC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CACvB,QAAgB,EAChB,IAAY,EACZ,aAAqB,MAAM;IAE3B,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB,EAAE;IAC9C,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAiB,EACjB,GAAW,EACX,SAAoC,EAAE;IAEtC,MAAM,SAAS,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;IAE9D,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,sBAAsB;IACtB,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;QAC5B,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAE5C,qBAAqB;IACrB,MAAM,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAElD,gBAAgB;IAChB,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAClC,SAAS,CAAC,SAAkC,EAC5C,GAAG,EACH,EAAE,EACF,EAAE,aAAa,EAAE,SAAS,CAAC,aAAa,EAAE,CAC3C,CAAC;IAEF,UAAU;IACV,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC;QAChC,MAAM,CAAC,KAAK,EAAE;KACf,CAAC,CAAC;IAEH,eAAe;IACf,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IAEpC,OAAO;QACL,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACzB,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;KACpC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,SAA0B,EAC1B,GAAW,EACX,SAAoC,EAAE;IAEtC,MAAM,SAAS,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;IAE9D,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,gBAAgB;IAChB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAEzD,kBAAkB;IAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,CACtC,SAAS,CAAC,SAAkC,EAC5C,GAAG,EACH,EAAE,EACF,EAAE,aAAa,EAAE,SAAS,CAAC,aAAa,EAAE,CAC3C,CAAC;IAEF,eAAe;IACf,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAE7B,UAAU;IACV,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;QAC3B,QAAQ,CAAC,KAAK,EAAE;KACjB,CAAC,CAAC;IAEH,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA2B,CAAC;IACnF,MAAM,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAEpC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAYD;;;GAGG;AACH,MAAM,OAAO,mBAAmB;IACtB,GAAG,CAAS;IAEpB,YAAY,GAAY;QACtB,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"pii-map-crypto.js","sourceRoot":"","sources":["../../src/crypto/pii-map-crypto.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAgBjC;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD,SAAS,EAAE,aAAa;IACxB,QAAQ,EAAE,EAAE;IACZ,aAAa,EAAE,EAAE;CAClB,CAAC;AAUF;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,UAAkC,EAAE;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;IACpC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CACvB,QAAgB,EAChB,IAAY,EACZ,aAAqB,MAAM;IAE3B,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB,EAAE;IAC9C,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAiB,EACjB,GAAW,EACX,SAAoC,EAAE;IAEtC,MAAM,SAAS,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;IAE9D,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,sBAAsB;IACtB,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;QAC5B,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAE5C,qBAAqB;IACrB,MAAM,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAElD,gBAAgB;IAChB,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAClC,SAAS,CAAC,SAAkC,EAC5C,GAAG,EACH,EAAE,EACF,EAAE,aAAa,EAAE,SAAS,CAAC,aAAa,EAAE,CAC3C,CAAC;IAEF,UAAU;IACV,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC;QAChC,MAAM,CAAC,KAAK,EAAE;KACf,CAAC,CAAC;IAEH,eAAe;IACf,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IAEpC,OAAO;QACL,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACzB,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;KACpC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,SAA0B,EAC1B,GAAW,EACX,SAAoC,EAAE;IAEtC,MAAM,SAAS,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;IAE9D,sBAAsB;IACtB,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,gBAAgB;IAChB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAEzD,kBAAkB;IAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,CACtC,SAAS,CAAC,SAAkC,EAC5C,GAAG,EACH,EAAE,EACF,EAAE,aAAa,EAAE,SAAS,CAAC,aAAa,EAAE,CAC3C,CAAC;IAEF,eAAe;IACf,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAE7B,UAAU;IACV,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;QAC3B,QAAQ,CAAC,KAAK,EAAE;KACjB,CAAC,CAAC;IAEH,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA2B,CAAC;IACnF,MAAM,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAEpC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAYD;;;GAGG;AACH,MAAM,OAAO,mBAAmB;IACtB,GAAG,CAAS;IAEpB,YAAY,GAAY;QACtB,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,CAAC;IAED,MAAM;QACJ,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,SAAS;QACP,IAAI,CAAC,GAAG,GAAG,WAAW,EAAE,CAAC;QACzB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,UAAU,CAAS;IAE3B,YAAY,aAAqB,oBAAoB;QACnD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,MAAM;QACJ,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qDAAqD,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC3G,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,UAAU,qBAAqB,CAAC,CAAC,CAAC;QACpG,CAAC;QAED,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,OAAO,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,CAAS,EAAE,CAAS;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtC,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
* Bridge Anonymization Module
|
|
3
3
|
* Main entry point for on-device PII anonymization
|
|
4
4
|
*/
|
|
5
|
-
export * from
|
|
6
|
-
export { Recognizer, RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from
|
|
7
|
-
export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, type INERModel, type NERModelConfig, type NERPrediction, type NERModelMode, type DownloadProgressCallback, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from
|
|
8
|
-
export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate, } from
|
|
9
|
-
export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, KeyProvider, InMemoryKeyProvider, EnvKeyProvider, } from
|
|
10
|
-
import { AnonymizationResult, AnonymizationPolicy } from
|
|
11
|
-
import { RecognizerRegistry } from
|
|
12
|
-
import { type INERModel } from
|
|
13
|
-
import { type NERModelMode, type DownloadProgressCallback } from
|
|
14
|
-
import { type KeyProvider } from
|
|
5
|
+
export * from "./types/index.js";
|
|
6
|
+
export { Recognizer, RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from "./recognizers/index.js";
|
|
7
|
+
export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, type INERModel, type NERModelConfig, type NERPrediction, type NERModelMode, type DownloadProgressCallback, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from "./ner/index.js";
|
|
8
|
+
export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate, enrichSemantics, inferGender, classifyLocation, getDatabaseStats, hasName, hasLocation, isSemanticDataAvailable, isSemanticDataDownloaded, getSemanticDataCacheDir, getDataDirectory, downloadSemanticData, ensureSemanticData, initializeSemanticData, loadSemanticData, clearSemanticData, clearSemanticDataCache, getSemanticDataInfo, SEMANTIC_DATA_FILES, extractTitle, extractTitlesFromSpans, mergeAdjacentTitleSpans, getTitlesForLanguage, getAllTitles, startsWithTitle, isOnlyTitle, type SemanticDataFileInfo, type EnricherConfig, type GenderResult, type LocationResult, type TitleExtractionResult, } from "./pipeline/index.js";
|
|
9
|
+
export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, KeyProvider, InMemoryKeyProvider, EnvKeyProvider, } from "./crypto/index.js";
|
|
10
|
+
import { AnonymizationResult, AnonymizationPolicy, SemanticConfig, PIIType } from "./types/index.js";
|
|
11
|
+
import { RecognizerRegistry } from "./recognizers/index.js";
|
|
12
|
+
import { type INERModel } from "./ner/index.js";
|
|
13
|
+
import { type NERModelMode, type DownloadProgressCallback } from "./ner/model-manager.js";
|
|
14
|
+
import { type KeyProvider } from "./crypto/index.js";
|
|
15
15
|
/**
|
|
16
16
|
* NER configuration options
|
|
17
17
|
*/
|
|
@@ -45,6 +45,12 @@ export interface NERConfig {
|
|
|
45
45
|
* Callback for status messages
|
|
46
46
|
*/
|
|
47
47
|
onStatus?: (status: string) => void;
|
|
48
|
+
/**
|
|
49
|
+
* Confidence thresholds per PII type (0.0 - 1.0)
|
|
50
|
+
* Overrides default thresholds for specified types
|
|
51
|
+
* @example { PERSON: 0.8, ORG: 0.7 }
|
|
52
|
+
*/
|
|
53
|
+
thresholds?: Partial<Record<PIIType, number>>;
|
|
48
54
|
}
|
|
49
55
|
/**
|
|
50
56
|
* Anonymizer configuration
|
|
@@ -58,9 +64,11 @@ export interface AnonymizerConfig {
|
|
|
58
64
|
*/
|
|
59
65
|
ner?: NERConfig;
|
|
60
66
|
/**
|
|
61
|
-
*
|
|
67
|
+
* Semantic enrichment configuration
|
|
68
|
+
* Enables MT-friendly PII tags with gender/scope attributes
|
|
69
|
+
* @default { enabled: false }
|
|
62
70
|
*/
|
|
63
|
-
|
|
71
|
+
semantic?: SemanticConfig;
|
|
64
72
|
/** Key provider for encryption (generates random key if not provided) */
|
|
65
73
|
keyProvider?: KeyProvider;
|
|
66
74
|
/** Default policy (uses default if not provided) */
|
|
@@ -78,15 +86,17 @@ export declare class Anonymizer {
|
|
|
78
86
|
private registry;
|
|
79
87
|
private nerModel;
|
|
80
88
|
private nerConfig;
|
|
89
|
+
private semanticConfig;
|
|
81
90
|
private keyProvider;
|
|
82
91
|
private defaultPolicy;
|
|
83
92
|
private modelVersion;
|
|
84
93
|
private policyVersion;
|
|
85
94
|
private initialized;
|
|
95
|
+
private semanticDataReady;
|
|
86
96
|
constructor(config?: AnonymizerConfig);
|
|
87
97
|
/**
|
|
88
98
|
* Initializes the anonymizer
|
|
89
|
-
* Downloads NER model if needed and loads
|
|
99
|
+
* Downloads NER model and semantic data if needed and loads them
|
|
90
100
|
*/
|
|
91
101
|
initialize(): Promise<void>;
|
|
92
102
|
/**
|
|
@@ -138,11 +148,6 @@ export declare class Anonymizer {
|
|
|
138
148
|
* ```
|
|
139
149
|
*/
|
|
140
150
|
export declare function createAnonymizer(config?: AnonymizerConfig): Anonymizer;
|
|
141
|
-
/**
|
|
142
|
-
* Creates an anonymizer with a custom NER model
|
|
143
|
-
* @deprecated Use createAnonymizer with ner: { mode: 'custom', modelPath, vocabPath } instead
|
|
144
|
-
*/
|
|
145
|
-
export declare function createAnonymizerWithNER(modelPath: string, vocabPath: string, config?: Omit<AnonymizerConfig, 'nerModel' | 'ner'>): Promise<Anonymizer>;
|
|
146
151
|
/**
|
|
147
152
|
* Convenience function for one-off anonymization
|
|
148
153
|
* Creates a temporary anonymizer with default settings (regex-only)
|
|
@@ -167,7 +172,7 @@ export declare function anonymizeRegexOnly(text: string, policy?: Partial<Anonym
|
|
|
167
172
|
* );
|
|
168
173
|
* ```
|
|
169
174
|
*/
|
|
170
|
-
export declare function anonymizeWithNER(text: string, nerConfig: Omit<NERConfig,
|
|
171
|
-
mode?:
|
|
175
|
+
export declare function anonymizeWithNER(text: string, nerConfig: Omit<NERConfig, "mode"> & {
|
|
176
|
+
mode?: "standard" | "quantized";
|
|
172
177
|
}, policy?: Partial<AnonymizationPolicy>): Promise<AnonymizationResult>;
|
|
173
178
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EACL,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EACL,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,WAAW,EAEX,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,EAEnB,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,eAAe,EACf,WAAW,EACX,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EACL,aAAa,EACb,aAAa,EACb,WAAW,EACX,SAAS,EACT,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,cAAc,GACf,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EAGnB,cAAc,EAEd,OAAO,EAGR,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAEL,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,SAAS,EAIf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,wBAAwB,EAC9B,MAAM,wBAAwB,CAAC;AAchC,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAG3B;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;OAMG;IACH,IAAI,EAAE,YAAY,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;IAE9C;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;OAGG;IACH,GAAG,CAAC,EAAE,SAAS,CAAC;IAEhB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B,yEAAyE;IACzE,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,oDAAoD;IACpD,aAAa,CAAC,EAAE,mBAAmB,CAAC;IAEpC,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,4BAA4B;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,QAAQ,CAA0B;IAC1C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,MAAM,GAAE,gBAAqB;IAsCzC;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAyFjC;;;;;;OAMG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC;IA8G/B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,WAAW,IAAI,kBAAkB;IAIjC;;OAEG;IACH,WAAW,IAAI,SAAS,GAAG,IAAI;IAI/B;;OAEG;IACH,IAAI,aAAa,IAAI,OAAO,CAE3B;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,UAAU,CAEtE;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAS9B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAQ9B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;CAAE,EACxE,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAe9B"}
|
package/dist/index.js
CHANGED
|
@@ -3,26 +3,33 @@
|
|
|
3
3
|
* Main entry point for on-device PII anonymization
|
|
4
4
|
*/
|
|
5
5
|
// Re-export types
|
|
6
|
-
export * from
|
|
6
|
+
export * from "./types/index.js";
|
|
7
7
|
// Re-export recognizers
|
|
8
|
-
export { RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from
|
|
8
|
+
export { RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from "./recognizers/index.js";
|
|
9
9
|
// Re-export NER components
|
|
10
|
-
export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from
|
|
10
|
+
export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from "./ner/index.js";
|
|
11
11
|
// Re-export pipeline components
|
|
12
|
-
export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate,
|
|
12
|
+
export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate, enrichSemantics, inferGender, classifyLocation, getDatabaseStats, hasName, hasLocation,
|
|
13
|
+
// Semantic data loader exports
|
|
14
|
+
isSemanticDataAvailable, isSemanticDataDownloaded, getSemanticDataCacheDir, getDataDirectory, downloadSemanticData, ensureSemanticData, initializeSemanticData, loadSemanticData, clearSemanticData, clearSemanticDataCache, getSemanticDataInfo, SEMANTIC_DATA_FILES,
|
|
15
|
+
// Title extractor exports
|
|
16
|
+
extractTitle, extractTitlesFromSpans, mergeAdjacentTitleSpans, getTitlesForLanguage, getAllTitles, startsWithTitle, isOnlyTitle, } from "./pipeline/index.js";
|
|
13
17
|
// Re-export crypto
|
|
14
|
-
export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, InMemoryKeyProvider, EnvKeyProvider, } from
|
|
18
|
+
export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, InMemoryKeyProvider, EnvKeyProvider, } from "./crypto/index.js";
|
|
15
19
|
// Main anonymization imports
|
|
16
|
-
import { createDefaultPolicy, mergePolicy, } from
|
|
17
|
-
import { createDefaultRegistry } from
|
|
18
|
-
import { NERModelStub, createNERModel, DEFAULT_LABEL_MAP } from
|
|
19
|
-
import { ensureModel } from
|
|
20
|
-
import { prenormalize } from
|
|
21
|
-
import { resolveEntities } from
|
|
22
|
-
import { tagEntities, countEntitiesByType } from
|
|
23
|
-
import { validateOutput } from
|
|
24
|
-
import {
|
|
25
|
-
import
|
|
20
|
+
import { createDefaultPolicy, mergePolicy, } from "./types/index.js";
|
|
21
|
+
import { createDefaultRegistry, } from "./recognizers/index.js";
|
|
22
|
+
import { NERModelStub, createNERModel, DEFAULT_LABEL_MAP, } from "./ner/index.js";
|
|
23
|
+
import { ensureModel, } from "./ner/model-manager.js";
|
|
24
|
+
import { prenormalize } from "./pipeline/prenormalize.js";
|
|
25
|
+
import { resolveEntities } from "./pipeline/resolver.js";
|
|
26
|
+
import { tagEntities, countEntitiesByType } from "./pipeline/tagger.js";
|
|
27
|
+
import { validateOutput } from "./pipeline/validator.js";
|
|
28
|
+
import { enrichSemantics } from "./pipeline/semantic-enricher.js";
|
|
29
|
+
import { ensureSemanticData, isSemanticDataAvailable, } from "./pipeline/semantic-data-loader.js";
|
|
30
|
+
import { extractTitlesFromSpans, mergeAdjacentTitleSpans, } from "./pipeline/title-extractor.js";
|
|
31
|
+
import { encryptPIIMap, generateKey, } from "./crypto/index.js";
|
|
32
|
+
import * as fs from "fs/promises";
|
|
26
33
|
/**
|
|
27
34
|
* Anonymizer instance
|
|
28
35
|
* Main class for performing PII anonymization
|
|
@@ -31,41 +38,60 @@ export class Anonymizer {
|
|
|
31
38
|
registry;
|
|
32
39
|
nerModel = null;
|
|
33
40
|
nerConfig;
|
|
41
|
+
semanticConfig;
|
|
34
42
|
keyProvider;
|
|
35
43
|
defaultPolicy;
|
|
36
44
|
modelVersion;
|
|
37
45
|
policyVersion;
|
|
38
46
|
initialized = false;
|
|
47
|
+
semanticDataReady = false;
|
|
39
48
|
constructor(config = {}) {
|
|
40
49
|
this.registry = config.registry ?? createDefaultRegistry();
|
|
41
50
|
this.keyProvider = config.keyProvider ?? null;
|
|
42
51
|
this.defaultPolicy = config.defaultPolicy ?? createDefaultPolicy();
|
|
43
|
-
this.policyVersion = config.policyVersion ??
|
|
52
|
+
this.policyVersion = config.policyVersion ?? "1.0.0";
|
|
44
53
|
// Handle NER configuration
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
54
|
+
this.nerConfig = config.ner ?? { mode: "disabled" };
|
|
55
|
+
this.modelVersion = config.modelVersion ?? "1.0.0";
|
|
56
|
+
// Merge NER thresholds into default policy if provided
|
|
57
|
+
if (this.nerConfig.thresholds !== undefined) {
|
|
58
|
+
const thresholdsMap = new Map(this.defaultPolicy.confidenceThresholds);
|
|
59
|
+
for (const [type, threshold] of Object.entries(this.nerConfig.thresholds)) {
|
|
60
|
+
if (threshold !== undefined) {
|
|
61
|
+
thresholdsMap.set(type, threshold);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
this.defaultPolicy = {
|
|
65
|
+
...this.defaultPolicy,
|
|
66
|
+
confidenceThresholds: thresholdsMap,
|
|
67
|
+
};
|
|
50
68
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
69
|
+
// Handle semantic configuration
|
|
70
|
+
this.semanticConfig = config.semantic ?? { enabled: false };
|
|
71
|
+
// If semantic is enabled, also enable it in the default policy
|
|
72
|
+
if (this.semanticConfig.enabled) {
|
|
73
|
+
this.defaultPolicy = {
|
|
74
|
+
...this.defaultPolicy,
|
|
75
|
+
enableSemanticMasking: true,
|
|
76
|
+
};
|
|
54
77
|
}
|
|
55
78
|
}
|
|
56
79
|
/**
|
|
57
80
|
* Initializes the anonymizer
|
|
58
|
-
* Downloads NER model if needed and loads
|
|
81
|
+
* Downloads NER model and semantic data if needed and loads them
|
|
59
82
|
*/
|
|
60
83
|
async initialize() {
|
|
61
84
|
if (this.initialized)
|
|
62
85
|
return;
|
|
63
86
|
// Handle NER model setup based on mode
|
|
64
|
-
if (this.nerConfig.mode ===
|
|
87
|
+
if (this.nerConfig.mode === "disabled") {
|
|
65
88
|
this.nerModel = new NERModelStub();
|
|
66
89
|
}
|
|
67
|
-
else if (this.nerConfig.mode ===
|
|
68
|
-
if (
|
|
90
|
+
else if (this.nerConfig.mode === "custom") {
|
|
91
|
+
if (this.nerConfig.modelPath === undefined ||
|
|
92
|
+
this.nerConfig.modelPath === "" ||
|
|
93
|
+
this.nerConfig.vocabPath === undefined ||
|
|
94
|
+
this.nerConfig.vocabPath === "") {
|
|
69
95
|
throw new Error("NER mode 'custom' requires modelPath and vocabPath");
|
|
70
96
|
}
|
|
71
97
|
this.nerModel = createNERModel({
|
|
@@ -84,7 +110,7 @@ export class Anonymizer {
|
|
|
84
110
|
// Load label map
|
|
85
111
|
let labelMap = DEFAULT_LABEL_MAP;
|
|
86
112
|
try {
|
|
87
|
-
const labelMapContent = await fs.readFile(labelMapPath,
|
|
113
|
+
const labelMapContent = await fs.readFile(labelMapPath, "utf-8");
|
|
88
114
|
labelMap = JSON.parse(labelMapContent);
|
|
89
115
|
}
|
|
90
116
|
catch {
|
|
@@ -97,11 +123,35 @@ export class Anonymizer {
|
|
|
97
123
|
modelVersion: this.modelVersion,
|
|
98
124
|
});
|
|
99
125
|
}
|
|
100
|
-
// Load the model
|
|
126
|
+
// Load the NER model
|
|
101
127
|
if (!this.nerModel.loaded) {
|
|
102
|
-
this.nerConfig.onStatus?.(
|
|
128
|
+
this.nerConfig.onStatus?.("Loading NER model...");
|
|
103
129
|
await this.nerModel.load();
|
|
104
|
-
this.nerConfig.onStatus?.(
|
|
130
|
+
this.nerConfig.onStatus?.("NER model loaded!");
|
|
131
|
+
}
|
|
132
|
+
// Handle semantic data setup if enabled
|
|
133
|
+
if (this.semanticConfig.enabled) {
|
|
134
|
+
const autoDownload = this.semanticConfig.autoDownload ?? true;
|
|
135
|
+
// Check if data is already available
|
|
136
|
+
if (!isSemanticDataAvailable()) {
|
|
137
|
+
if (!autoDownload) {
|
|
138
|
+
throw new Error("Semantic masking is enabled but data files are not available.\n\n" +
|
|
139
|
+
"To download automatically, use:\n" +
|
|
140
|
+
" createAnonymizer({ semantic: { enabled: true, autoDownload: true } })\n\n" +
|
|
141
|
+
"Or disable semantic masking:\n" +
|
|
142
|
+
" createAnonymizer({ semantic: { enabled: false } })");
|
|
143
|
+
}
|
|
144
|
+
// Download semantic data
|
|
145
|
+
await ensureSemanticData({
|
|
146
|
+
autoDownload: true,
|
|
147
|
+
onProgress: this.semanticConfig.onDownloadProgress,
|
|
148
|
+
onStatus: this.semanticConfig.onStatus,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
this.semanticConfig.onStatus?.("Semantic data already cached");
|
|
153
|
+
}
|
|
154
|
+
this.semanticDataReady = true;
|
|
105
155
|
}
|
|
106
156
|
this.modelVersion = this.nerModel.version;
|
|
107
157
|
this.initialized = true;
|
|
@@ -129,8 +179,25 @@ export class Anonymizer {
|
|
|
129
179
|
const nerMatches = nerResult.spans;
|
|
130
180
|
// Step 4: Resolve and merge entities
|
|
131
181
|
const resolvedMatches = resolveEntities(regexMatches, nerMatches, effectivePolicy, normalizedText);
|
|
182
|
+
// Step 4.5: Merge adjacent title+name PERSON spans (if semantic masking enabled)
|
|
183
|
+
// This fixes NER models that split "Mrs. Smith" into two entities
|
|
184
|
+
const mergedMatches = effectivePolicy.enableSemanticMasking === true
|
|
185
|
+
? mergeAdjacentTitleSpans(resolvedMatches, normalizedText)
|
|
186
|
+
: resolvedMatches;
|
|
187
|
+
// Step 4.6: Extract titles from PERSON entities (if semantic masking enabled)
|
|
188
|
+
// This strips honorific titles (Dr., Mrs., etc.) so they remain visible for translation
|
|
189
|
+
const titleExtractedMatches = effectivePolicy.enableSemanticMasking
|
|
190
|
+
? extractTitlesFromSpans(mergedMatches, normalizedText)
|
|
191
|
+
: mergedMatches;
|
|
192
|
+
// Step 4.6: Enrich with semantic attributes (if enabled)
|
|
193
|
+
// This adds gender for PERSON and scope for LOCATION entities
|
|
194
|
+
const enrichedMatches = effectivePolicy.enableSemanticMasking
|
|
195
|
+
? enrichSemantics(titleExtractedMatches, {
|
|
196
|
+
locale: locale !== undefined ? locale.split("-")[0] : undefined, // Extract language code
|
|
197
|
+
})
|
|
198
|
+
: titleExtractedMatches;
|
|
132
199
|
// Step 5: Tag entities and build PII map
|
|
133
|
-
const { anonymizedText, entities, piiMap } = tagEntities(normalizedText,
|
|
200
|
+
const { anonymizedText, entities, piiMap } = tagEntities(normalizedText, enrichedMatches, effectivePolicy);
|
|
134
201
|
// Step 6: Validate output
|
|
135
202
|
const validation = validateOutput(anonymizedText, entities, Array.from(piiMap.keys()), effectivePolicy);
|
|
136
203
|
if (!validation.valid) {
|
|
@@ -140,7 +207,7 @@ export class Anonymizer {
|
|
|
140
207
|
message: e.message,
|
|
141
208
|
}));
|
|
142
209
|
// eslint-disable-next-line no-console
|
|
143
|
-
console.warn(
|
|
210
|
+
console.warn("Validation warnings:", safeErrors);
|
|
144
211
|
}
|
|
145
212
|
// Step 7: Encrypt PII map
|
|
146
213
|
const encryptionKey = this.keyProvider !== null
|
|
@@ -158,7 +225,7 @@ export class Anonymizer {
|
|
|
158
225
|
leakScanPassed: validation.leakScanPassed,
|
|
159
226
|
};
|
|
160
227
|
// Step 9: Build result (without original text in entities)
|
|
161
|
-
const safeEntities = entities.map(({ original, ...rest }) => rest);
|
|
228
|
+
const safeEntities = entities.map(({ original: _original, ...rest }) => rest);
|
|
162
229
|
return {
|
|
163
230
|
anonymizedText,
|
|
164
231
|
entities: safeEntities,
|
|
@@ -220,22 +287,6 @@ export class Anonymizer {
|
|
|
220
287
|
export function createAnonymizer(config) {
|
|
221
288
|
return new Anonymizer(config);
|
|
222
289
|
}
|
|
223
|
-
/**
|
|
224
|
-
* Creates an anonymizer with a custom NER model
|
|
225
|
-
* @deprecated Use createAnonymizer with ner: { mode: 'custom', modelPath, vocabPath } instead
|
|
226
|
-
*/
|
|
227
|
-
export async function createAnonymizerWithNER(modelPath, vocabPath, config) {
|
|
228
|
-
const anonymizer = new Anonymizer({
|
|
229
|
-
...config,
|
|
230
|
-
ner: {
|
|
231
|
-
mode: 'custom',
|
|
232
|
-
modelPath,
|
|
233
|
-
vocabPath,
|
|
234
|
-
},
|
|
235
|
-
});
|
|
236
|
-
await anonymizer.initialize();
|
|
237
|
-
return anonymizer;
|
|
238
|
-
}
|
|
239
290
|
/**
|
|
240
291
|
* Convenience function for one-off anonymization
|
|
241
292
|
* Creates a temporary anonymizer with default settings (regex-only)
|
|
@@ -279,7 +330,7 @@ export async function anonymizeRegexOnly(text, policy) {
|
|
|
279
330
|
export async function anonymizeWithNER(text, nerConfig, policy) {
|
|
280
331
|
const anonymizer = createAnonymizer({
|
|
281
332
|
ner: {
|
|
282
|
-
mode: nerConfig.mode ??
|
|
333
|
+
mode: nerConfig.mode ?? "quantized",
|
|
283
334
|
...nerConfig,
|
|
284
335
|
},
|
|
285
336
|
});
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,kBAAkB;AAClB,cAAc,kBAAkB,CAAC;AAEjC,wBAAwB;AACxB,OAAO,EAEL,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAEhC,2BAA2B;AAC3B,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EAMd,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAExB,gCAAgC;AAChC,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,kBAAkB;AAClB,cAAc,kBAAkB,CAAC;AAEjC,wBAAwB;AACxB,OAAO,EAEL,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAEhC,2BAA2B;AAC3B,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EAMd,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAExB,gCAAgC;AAChC,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,WAAW;AACX,+BAA+B;AAC/B,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB;AACnB,0BAA0B;AAC1B,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,eAAe,EACf,WAAW,GAMZ,MAAM,qBAAqB,CAAC;AAE7B,mBAAmB;AACnB,OAAO,EACL,aAAa,EACb,aAAa,EACb,WAAW,EACX,SAAS,EACT,YAAY,EAEZ,mBAAmB,EACnB,cAAc,GACf,MAAM,mBAAmB,CAAC;AAE3B,6BAA6B;AAC7B,OAAO,EAQL,mBAAmB,EACnB,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,qBAAqB,GAEtB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,YAAY,EACZ,cAAc,EACd,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEL,WAAW,GAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EACL,kBAAkB,EAClB,uBAAuB,GACxB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACL,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,aAAa,EACb,WAAW,GAEZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAkFlC;;;GAGG;AACH,MAAM,OAAO,UAAU;IACb,QAAQ,CAAqB;IAC7B,QAAQ,GAAqB,IAAI,CAAC;IAClC,SAAS,CAAY;IACrB,cAAc,CAAiB;IAC/B,WAAW,CAAqB;IAChC,aAAa,CAAsB;IACnC,YAAY,CAAS;IACrB,aAAa,CAAS;IACtB,WAAW,GAAG,KAAK,CAAC;IACpB,iBAAiB,GAAG,KAAK,CAAC;IAElC,YAAY,SAA2B,EAAE;QACvC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC3D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,mBAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,OAAO,CAAC;QAErD,2BAA2B;QAC3B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC;QAEnD,uDAAuD;QACvD,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;YACvE,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAC5C,IAAI,CAAC,SAAS,CAAC,UAAU,CAC1B,EAAE,CAAC;gBACF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,aAAa,CAAC,GAAG,CAAC,IAAe,EAAE,SAAS,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,oBAAoB,EAAE,aAAa;aACpC,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAE5D,+DAA+D;QAC/D,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,qBAAqB,EAAE,IAAI;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,uCAAuC;QACvC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAC;QACrC,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5C,IACE,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE;gBAC/B,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE,EAC/B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACxE,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,WAAW,CAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,EACnB;gBACE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,IAAI;gBACjD,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC7C,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ;aAClC,CACF,CAAC;YAEF,iBAAiB;YACjB,IAAI,QAAQ,GAAG,iBAAiB,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;gBACjE,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAa,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS;gBACT,SAAS;gBACT,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACL,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,sBAAsB,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,mBAAmB,CAAC,CAAC;QACjD,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,IAAI,IAAI,CAAC;YAE9D,qCAAqC;YACrC,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC;gBAC/B,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,mEAAmE;wBACjE,mCAAmC;wBACnC,6EAA6E;wBAC7E,gCAAgC;wBAChC,sDAAsD,CACzD,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,MAAM,kBAAkB,CAAC;oBACvB,YAAY,EAAE,IAAI;oBAClB,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,kBAAkB;oBAClD,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;iBACvC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,8BAA8B,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAe,EACf,MAAqC;QAErC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,6BAA6B;QAC7B,MAAM,eAAe,GACnB,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QAElE,6BAA6B;QAC7B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAE1C,gCAAgC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;QAE5E,wBAAwB;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,OAAO,CAC5C,cAAc,EACd,eAAe,CAChB,CAAC;QACF,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC;QAEnC,qCAAqC;QACrC,MAAM,eAAe,GAAG,eAAe,CACrC,YAAY,EACZ,UAAU,EACV,eAAe,EACf,cAAc,CACf,CAAC;QAEF,iFAAiF;QACjF,kEAAkE;QAClE,MAAM,aAAa,GACjB,eAAe,CAAC,qBAAqB,KAAK,IAAI;YAC5C,CAAC,CAAC,uBAAuB,CAAC,eAAe,EAAE,cAAc,CAAC;YAC1D,CAAC,CAAC,eAAe,CAAC;QAEtB,8EAA8E;QAC9E,wFAAwF;QACxF,MAAM,qBAAqB,GAAG,eAAe,CAAC,qBAAqB;YACjE,CAAC,CAAC,sBAAsB,CAAC,aAAa,EAAE,cAAc,CAAC;YACvD,CAAC,CAAC,aAAa,CAAC;QAElB,yDAAyD;QACzD,8DAA8D;QAC9D,MAAM,eAAe,GAAG,eAAe,CAAC,qBAAqB;YAC3D,CAAC,CAAC,eAAe,CAAC,qBAAqB,EAAE;gBACrC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,wBAAwB;aAC1F,CAAC;YACJ,CAAC,CAAC,qBAAqB,CAAC;QAE1B,yCAAyC;QACzC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,CACtD,cAAc,EACd,eAAe,EACf,eAAe,CAChB,CAAC;QAEF,0BAA0B;QAC1B,MAAM,UAAU,GAAG,cAAc,CAC/B,cAAc,EACd,QAAQ,EACR,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EACzB,eAAe,CAChB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACtB,mDAAmD;YACnD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/C,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC,CAAC;YACJ,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;QACnD,CAAC;QAED,0BAA0B;QAC1B,MAAM,aAAa,GACjB,IAAI,CAAC,WAAW,KAAK,IAAI;YACvB,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;YACjC,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpB,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE7D,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,KAAK,GAAuB;YAChC,YAAY,EAAE,mBAAmB,CAAC,QAAQ,CAAC;YAC3C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,cAAc,EAAE,UAAU,CAAC,cAAc;SAC1C,CAAC;QAEF,2DAA2D;QAC3D,MAAM,YAAY,GAAuC,QAAQ,CAAC,GAAG,CACnE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAC3C,CAAC;QAEF,OAAO;YACL,cAAc;YACd,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,eAAe;YACvB,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,MAAe,EACf,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAqC;IAErC,kCAAkC;IAClC,MAAM,eAAe,GAAiC;QACpD,GAAG,MAAM;QACT,eAAe,EAAE,IAAI,GAAG,EAAE,EAAE,wBAAwB;KACrD,CAAC;IAEF,OAAO,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,SAAwE,EACxE,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,CAAC;QAClC,GAAG,EAAE;YACH,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,WAAW;YACnC,GAAG,SAAS;SACb;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,UAAU,GAAG,WAAW,EAAE,SAAS,CAyBtE,CAAC;AAYF;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAYzC;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM,CAEnE;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAkBxF;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,KAAK,IAAI,CAAC;AAyEX;;GAEG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,UAAU,CAAC,EAAE,wBAAwB,EACrC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC,CAiDjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,wBAAwB,CAAC;IACtC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAiCzE;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAQpF;AAED;;GAEG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,UAAU,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAY3H;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAEtE"}
|