euredact 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +296 -43
- package/dist/cjs/index.js +4 -0
- package/dist/cjs/rules/countries/index.js +50 -0
- package/dist/cjs/rules/engine.js +60 -19
- package/dist/cjs/rules/matchers.js +5 -0
- package/dist/cjs/rules/validators.js +30 -0
- package/dist/cjs/sdk.js +10 -6
- package/dist/cjs/types.js +1 -0
- package/dist/esm/index.d.ts +1 -0
- package/dist/esm/index.js +3 -0
- package/dist/esm/rules/countries/index.js +50 -0
- package/dist/esm/rules/engine.d.ts +6 -0
- package/dist/esm/rules/engine.js +60 -19
- package/dist/esm/rules/matchers.d.ts +2 -1
- package/dist/esm/rules/matchers.js +5 -0
- package/dist/esm/rules/validators.js +30 -0
- package/dist/esm/sdk.d.ts +3 -2
- package/dist/esm/sdk.js +10 -6
- package/dist/esm/types.d.ts +3 -2
- package/dist/esm/types.js +1 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
# euredact
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**European PII detection and redaction for Node.js / TypeScript**
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Fast, zero-dependency European PII detection and redaction SDK. Detects and
|
|
8
|
+
redacts personal data across **31 European countries** using rule-based pattern
|
|
9
|
+
matching with checksum validation, context-aware false-positive suppression,
|
|
10
|
+
and priority-based deduplication.
|
|
6
11
|
|
|
7
12
|
## Install
|
|
8
13
|
|
|
@@ -26,64 +31,304 @@ console.log(result.detections);
|
|
|
26
31
|
// [{ entityType: "NATIONAL_ID", text: "123456782", start: 12, end: 21, ... }, ...]
|
|
27
32
|
```
|
|
28
33
|
|
|
29
|
-
##
|
|
34
|
+
## Features
|
|
35
|
+
|
|
36
|
+
- **31 European countries** with country-specific patterns
|
|
37
|
+
- **25+ PII entity types:** national IDs, IBANs, phone numbers, email, VAT
|
|
38
|
+
numbers, license plates, credit cards, BIC/SWIFT, VIN, IMEI, GPS coordinates,
|
|
39
|
+
UUIDs, social handles, IP/IPv6, MAC addresses, secrets/API keys, and more
|
|
40
|
+
- **Secret/API key detection:** known-prefix patterns for AWS, GitHub, Stripe,
|
|
41
|
+
OpenAI, Slack, JWT, SendGrid, plus Shannon entropy-based detection for generic
|
|
42
|
+
high-entropy tokens near context keywords
|
|
43
|
+
- **Custom patterns:** register your own regex patterns for domain-specific PII
|
|
44
|
+
- **Checksum validation:** IBAN mod-97, Luhn (credit cards/IMEI), and 30+
|
|
45
|
+
country-specific validators
|
|
46
|
+
- **Priority-aware deduplication:** validated (checksum) > custom > regex-only
|
|
47
|
+
- **Context-aware:** keyword proximity checks and structural detection (JSON field
|
|
48
|
+
names, CSV headers) for ambiguous patterns
|
|
49
|
+
- **Zero runtime dependencies**
|
|
50
|
+
- **ESM and CommonJS** dual-published
|
|
51
|
+
|
|
52
|
+
### Supported Countries
|
|
30
53
|
|
|
31
|
-
|
|
54
|
+
| Region | Countries |
|
|
55
|
+
|---|---|
|
|
56
|
+
| Western Europe | AT, BE, CH, DE, FR, LU, NL |
|
|
57
|
+
| Southern Europe | CY, EL, ES, IT, MT, PT |
|
|
58
|
+
| Northern Europe | DK, EE, FI, IS, LT, LV, NO, SE |
|
|
59
|
+
| Eastern Europe | BG, CZ, HR, HU, PL, RO, SI, SK |
|
|
60
|
+
| British Isles | IE, UK |
|
|
32
61
|
|
|
33
|
-
|
|
62
|
+
## API Reference
|
|
63
|
+
|
|
64
|
+
### Module-Level Functions
|
|
65
|
+
|
|
66
|
+
#### `redact(text, options?)`
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
function redact(text: string, options?: RedactOptions): RedactResult;
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Main entry point. Detects and redacts PII in the given text.
|
|
34
73
|
|
|
35
74
|
```ts
|
|
36
75
|
interface RedactOptions {
|
|
37
|
-
countries?: string[];
|
|
38
|
-
|
|
39
|
-
detectDates?: boolean;
|
|
40
|
-
cache?: boolean;
|
|
76
|
+
countries?: string[] | null; // Country codes (e.g. ["NL", "BE"]) — null loads all
|
|
77
|
+
referentialIntegrity?: boolean; // Replace with consistent labels (default: false)
|
|
78
|
+
detectDates?: boolean; // Include DOB/date-of-death detections (default: false)
|
|
79
|
+
cache?: boolean; // Enable result caching (default: true)
|
|
41
80
|
}
|
|
42
81
|
```
|
|
43
82
|
|
|
44
|
-
|
|
83
|
+
| Parameter | Default | Description |
|
|
84
|
+
|---|---|---|
|
|
85
|
+
| `countries` | `null` | ISO 3166-1 alpha-2 codes to restrict detection. `null` loads all 31 countries. |
|
|
86
|
+
| `referentialIntegrity` | `false` | Replace PII with consistent labels instead of entity-type labels. |
|
|
87
|
+
| `detectDates` | `false` | Include date-of-birth and date-of-death detections. Off by default. |
|
|
88
|
+
| `cache` | `true` | Cache results for identical inputs. |
|
|
89
|
+
|
|
90
|
+
#### `redactBatch(texts, options?)`
|
|
91
|
+
|
|
92
|
+
```ts
|
|
93
|
+
function redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Process multiple texts efficiently. Loads country configs once. Returns results
|
|
97
|
+
in the same order as the input.
|
|
45
98
|
|
|
46
|
-
|
|
99
|
+
#### `addCustomPattern(name, pattern)`
|
|
47
100
|
|
|
48
|
-
|
|
101
|
+
```ts
|
|
102
|
+
function addCustomPattern(name: string, pattern: string): void;
|
|
103
|
+
```
|
|
49
104
|
|
|
50
|
-
|
|
105
|
+
Register a custom regex pattern. Matches are reported with `name` as the entity
|
|
106
|
+
type. See [Custom Patterns](#custom-patterns) below.
|
|
51
107
|
|
|
52
|
-
|
|
108
|
+
#### `availableCountries()`
|
|
53
109
|
|
|
54
|
-
|
|
110
|
+
```ts
|
|
111
|
+
function availableCountries(): string[];
|
|
112
|
+
```
|
|
55
113
|
|
|
56
|
-
|
|
114
|
+
Returns a sorted list of supported ISO country codes.
|
|
57
115
|
|
|
58
|
-
|
|
59
|
-
|----------|---------|
|
|
60
|
-
| `NATIONAL_ID` | BSN, Personalausweis, NIR, CPR, personnummer, ... |
|
|
61
|
-
| `IBAN` | Country-specific IBAN with ISO 13616 checksum |
|
|
62
|
-
| `PHONE` | National and international formats per country |
|
|
63
|
-
| `EMAIL` | RFC 5322 simplified |
|
|
64
|
-
| `POSTAL_CODE` | Country-specific formats with context validation |
|
|
65
|
-
| `TAX_ID` | Steuer-ID, NIF, NIP, ... with checksums |
|
|
66
|
-
| `CREDIT_CARD` | Visa, Mastercard, Amex with Luhn check |
|
|
67
|
-
| `LICENSE_PLATE` | Country-specific formats |
|
|
68
|
-
| `PASSPORT` | Context-required per country |
|
|
69
|
-
| `VAT` | EU VAT numbers |
|
|
70
|
-
| `BIC` | SWIFT/BIC codes |
|
|
71
|
-
| `VIN` | ISO 3779 vehicle identification |
|
|
72
|
-
| `IP_ADDRESS` | IPv4 and IPv6 |
|
|
73
|
-
| `MAC_ADDRESS` | Colon, dash, and Cisco dot notation |
|
|
74
|
-
| `UUID` | RFC 4122 versions 1-5 |
|
|
75
|
-
| `IMEI` | 15-digit with Luhn check |
|
|
76
|
-
| `GPS_COORDINATES` | Decimal degrees with 4+ decimal places |
|
|
77
|
-
| `SOCIAL_HANDLE` | @username format |
|
|
78
|
-
| `DOB` | Date of birth (context-required, opt-in) |
|
|
116
|
+
### Instance-Based API (`EuRedact` Class)
|
|
79
117
|
|
|
80
|
-
|
|
118
|
+
For applications that need isolated instances with separate caches and custom
|
|
119
|
+
pattern registrations:
|
|
81
120
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
121
|
+
```ts
|
|
122
|
+
import { EuRedact } from "euredact";
|
|
123
|
+
|
|
124
|
+
const instance = new EuRedact();
|
|
125
|
+
instance.addCustomPattern("CASE_REF", "CASE-\\d{8}");
|
|
126
|
+
|
|
127
|
+
const result = instance.redact("See CASE-20260401 for details", {
|
|
128
|
+
countries: ["NL", "BE"],
|
|
129
|
+
});
|
|
130
|
+
console.log(result.redactedText);
|
|
131
|
+
// "See [CASE_REF] for details"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The `EuRedact` class exposes: `redact()`, `redactBatch()`, and
|
|
135
|
+
`addCustomPattern()`.
|
|
136
|
+
|
|
137
|
+
### Return Types
|
|
138
|
+
|
|
139
|
+
#### `RedactResult`
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
interface RedactResult {
|
|
143
|
+
redactedText: string; // The input text with PII replaced
|
|
144
|
+
detections: Detection[]; // All PII spans found
|
|
145
|
+
source: string; // Detection backend ("rules")
|
|
146
|
+
degraded: boolean; // True if the engine fell back to a simpler mode
|
|
147
|
+
}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
#### `Detection`
|
|
151
|
+
|
|
152
|
+
```ts
|
|
153
|
+
interface Detection {
|
|
154
|
+
entityType: EntityType | string; // PII category (enum or custom name)
|
|
155
|
+
start: number; // Start offset in the original text
|
|
156
|
+
end: number; // End offset (exclusive)
|
|
157
|
+
text: string; // The matched substring
|
|
158
|
+
source: DetectionSource; // "rules" or "cloud"
|
|
159
|
+
country: string | null; // ISO code or null for shared/custom patterns
|
|
160
|
+
confidence: string; // Confidence level
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
#### `EntityType`
|
|
165
|
+
|
|
166
|
+
String enum with all supported PII categories:
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
NAME ADDRESS IBAN BIC
|
|
170
|
+
CREDIT_CARD PHONE EMAIL DOB
|
|
171
|
+
DATE_OF_DEATH NATIONAL_ID SSN TAX_ID
|
|
172
|
+
PASSPORT DRIVERS_LICENSE RESIDENCE_PERMIT LICENSE_PLATE
|
|
173
|
+
VIN VAT POSTAL_CODE IP_ADDRESS
|
|
174
|
+
IPV6_ADDRESS MAC_ADDRESS HEALTH_INSURANCE HEALTHCARE_PROVIDER
|
|
175
|
+
CHAMBER_OF_COMMERCE IMEI GPS_COORDINATES UUID
|
|
176
|
+
SOCIAL_HANDLE SECRET OTHER
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
For custom patterns, `entityType` is a plain string (e.g. `"EMPLOYEE_ID"`).
|
|
180
|
+
|
|
181
|
+
## Custom Patterns
|
|
182
|
+
|
|
183
|
+
Register domain-specific PII patterns at runtime:
|
|
184
|
+
|
|
185
|
+
```ts
|
|
186
|
+
import { addCustomPattern, redact } from "euredact";
|
|
187
|
+
|
|
188
|
+
// Register patterns
|
|
189
|
+
addCustomPattern("EMPLOYEE_ID", "EMP-\\d{6}");
|
|
190
|
+
addCustomPattern("CASE_REF", "CASE-\\d{8}");
|
|
191
|
+
|
|
192
|
+
// They are detected alongside built-in PII
|
|
193
|
+
const result = redact(
|
|
194
|
+
"Employee EMP-123456, email jan@example.com, ref CASE-20260401"
|
|
195
|
+
);
|
|
196
|
+
console.log(result.redactedText);
|
|
197
|
+
// "Employee [EMPLOYEE_ID], email [EMAIL], ref [CASE_REF]"
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### How Custom Patterns Work
|
|
201
|
+
|
|
202
|
+
- `name` becomes the entity type in detections and replacement tags
|
|
203
|
+
- `pattern` is a JavaScript regular expression (same syntax as `RegExp`)
|
|
204
|
+
- Custom patterns are always active regardless of the `countries` option
|
|
205
|
+
- In overlap resolution, custom patterns have higher priority than built-in
|
|
206
|
+
regex-only patterns but lower priority than built-in patterns with a passing
|
|
207
|
+
checksum validator
|
|
208
|
+
|
|
209
|
+
### Instance Isolation
|
|
210
|
+
|
|
211
|
+
Custom patterns registered on the module-level function apply to the shared
|
|
212
|
+
singleton. For isolated registrations, use separate `EuRedact` instances:
|
|
213
|
+
|
|
214
|
+
```ts
|
|
215
|
+
import { EuRedact } from "euredact";
|
|
216
|
+
|
|
217
|
+
const a = new EuRedact();
|
|
218
|
+
a.addCustomPattern("EMPLOYEE_ID", "EMP-\\d{6}");
|
|
219
|
+
|
|
220
|
+
const b = new EuRedact();
|
|
221
|
+
b.addCustomPattern("CASE_REF", "CASE-\\d{8}");
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## Secret and API Key Detection
|
|
225
|
+
|
|
226
|
+
Built-in detection for API keys, tokens, and passwords. Always active -- no
|
|
227
|
+
configuration required.
|
|
228
|
+
|
|
229
|
+
### Known-Prefix Patterns
|
|
230
|
+
|
|
231
|
+
| Pattern | Description |
|
|
232
|
+
|---|---|
|
|
233
|
+
| `AKIA...` | AWS Access Key ID |
|
|
234
|
+
| `ghp_`, `gho_`, `ghs_`, `github_pat_` | GitHub tokens (PAT, OAuth, app, server) |
|
|
235
|
+
| `sk_live_`, `pk_live_`, `sk_test_`, `pk_test_` | Stripe secret and publishable keys |
|
|
236
|
+
| `sk-`, `sk-ant-` | OpenAI and Anthropic API keys |
|
|
237
|
+
| `xoxb-`, `xoxp-`, `xoxa-`, `xoxs-` | Slack tokens |
|
|
238
|
+
| `eyJ...` (3-part base64url) | JWT tokens |
|
|
239
|
+
| `SG.` | SendGrid API keys |
|
|
240
|
+
|
|
241
|
+
```ts
|
|
242
|
+
const result = redact("My API key is sk-proj-abc123def456ghi789jkl0");
|
|
243
|
+
console.log(result.redactedText);
|
|
244
|
+
// "My API key is [SECRET]"
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
### Entropy-Based Detection
|
|
248
|
+
|
|
249
|
+
High-entropy strings (32+ alphanumeric characters) near context keywords like
|
|
250
|
+
`key`, `token`, `secret`, `password`, `credential`, `auth`, or `bearer`
|
|
251
|
+
(including 12 European language translations) are flagged via Shannon entropy
|
|
252
|
+
analysis:
|
|
253
|
+
|
|
254
|
+
```ts
|
|
255
|
+
redact("The api_key is xK9mPqR7vLnW2bFjY8cGhT4sDfAeU6iO").redactedText;
|
|
256
|
+
// "The api_key is [SECRET]"
|
|
257
|
+
|
|
258
|
+
// Without a context keyword -- not flagged:
|
|
259
|
+
redact("identifier: xK9mPqR7vLnW2bFjY8cGhT4sDfAeU6iO").redactedText;
|
|
260
|
+
// "identifier: xK9mPqR7vLnW2bFjY8cGhT4sDfAeU6iO"
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
## Country Hints
|
|
264
|
+
|
|
265
|
+
When you know which countries appear in your data, pass them explicitly:
|
|
266
|
+
|
|
267
|
+
```ts
|
|
268
|
+
const result = redact(text, { countries: ["NL", "BE"] });
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
This restricts pattern matching to Dutch and Belgian rules (plus shared patterns
|
|
272
|
+
and custom patterns). Benefits:
|
|
273
|
+
|
|
274
|
+
- **Fewer false positives.** A 9-digit number that passes the Dutch BSN checksum
|
|
275
|
+
won't also be tested against unrelated country patterns.
|
|
276
|
+
- **Faster.** Fewer patterns to compile and scan.
|
|
277
|
+
|
|
278
|
+
## Referential Integrity
|
|
279
|
+
|
|
280
|
+
When `referentialIntegrity: true`, each unique PII value is mapped to a consistent
|
|
281
|
+
label:
|
|
282
|
+
|
|
283
|
+
```ts
|
|
284
|
+
const result = redact(
|
|
285
|
+
"BSN 123456782 en later weer 123456782",
|
|
286
|
+
{ countries: ["NL"], referentialIntegrity: true }
|
|
287
|
+
);
|
|
288
|
+
console.log(result.redactedText);
|
|
289
|
+
// "BSN NATIONAL_ID_1 en later weer NATIONAL_ID_1"
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
## Architecture
|
|
293
|
+
|
|
294
|
+
```
|
|
295
|
+
Input text
|
|
296
|
+
|
|
|
297
|
+
v
|
|
298
|
+
[Normalizer] -- Unicode normalization, whitespace cleanup
|
|
299
|
+
|
|
|
300
|
+
v
|
|
301
|
+
[Pass 1: Pattern Matching] -- All country + shared + custom regexes
|
|
302
|
+
|
|
|
303
|
+
v
|
|
304
|
+
[Pass 2a: Validation] -- Checksum validators (mod-97, Luhn, entropy, ...)
|
|
305
|
+
| Failed-validation spans become suppression zones
|
|
306
|
+
v
|
|
307
|
+
[Pass 2b: Suppression] -- Remove false positives (currency amounts, units,
|
|
308
|
+
| references, overlapping matches in suppression zones)
|
|
309
|
+
v
|
|
310
|
+
[Deduplication] -- Priority-aware: validated > custom > regex-only
|
|
311
|
+
| Within same tier, longer span wins
|
|
312
|
+
v
|
|
313
|
+
[Replacement] -- Right-to-left substitution with [ENTITY_TYPE] labels
|
|
314
|
+
| or labels
|
|
315
|
+
v
|
|
316
|
+
RedactResult
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
### Suppression Zones
|
|
320
|
+
|
|
321
|
+
When a regex matches a pattern that has a checksum validator but the checksum
|
|
322
|
+
fails, the matched span becomes a "suppression zone." Any purely regex-based
|
|
323
|
+
detection fully contained within that zone is suppressed as a false positive.
|
|
324
|
+
|
|
325
|
+
### Deduplication Priority
|
|
326
|
+
|
|
327
|
+
1. **Validated patterns** (checksum passes) -- highest priority
|
|
328
|
+
2. **Custom patterns** (registered via `addCustomPattern()`)
|
|
329
|
+
3. **Regex-only patterns** (no validator) -- lowest priority
|
|
330
|
+
|
|
331
|
+
Within the same tier, the longer span wins.
|
|
87
332
|
|
|
88
333
|
## CommonJS
|
|
89
334
|
|
|
@@ -91,6 +336,14 @@ AT, BE, BG, CH, CY, CZ, DE, DK, EE, EL, ES, FI, FR, HR, HU, IE, IS, IT, LT, LU,
|
|
|
91
336
|
const { redact } = require("euredact");
|
|
92
337
|
```
|
|
93
338
|
|
|
339
|
+
## Performance
|
|
340
|
+
|
|
341
|
+
| Metric | Value |
|
|
342
|
+
|---|---|
|
|
343
|
+
| Latency per redaction | ~0.02 ms |
|
|
344
|
+
| Package size | ~86 KB |
|
|
345
|
+
| Runtime dependencies | 0 |
|
|
346
|
+
|
|
94
347
|
## License
|
|
95
348
|
|
|
96
349
|
Apache-2.0
|
package/dist/cjs/index.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.COUNTRY_CONFIGS = exports.EuRedact = exports.DetectionSource = exports.EntityType = void 0;
|
|
4
4
|
exports.availableCountries = availableCountries;
|
|
5
5
|
exports.redact = redact;
|
|
6
|
+
exports.addCustomPattern = addCustomPattern;
|
|
6
7
|
exports.redactBatch = redactBatch;
|
|
7
8
|
var types_js_1 = require("./types.js");
|
|
8
9
|
Object.defineProperty(exports, "EntityType", { enumerable: true, get: function () { return types_js_1.EntityType; } });
|
|
@@ -25,6 +26,9 @@ function availableCountries() {
|
|
|
25
26
|
function redact(text, options) {
|
|
26
27
|
return getInstance().redact(text, options);
|
|
27
28
|
}
|
|
29
|
+
function addCustomPattern(name, pattern) {
|
|
30
|
+
getInstance().addCustomPattern(name, pattern);
|
|
31
|
+
}
|
|
28
32
|
function redactBatch(texts, options) {
|
|
29
33
|
return getInstance().redactBatch(texts, options);
|
|
30
34
|
}
|
|
@@ -49,6 +49,56 @@ const SHARED = {
|
|
|
49
49
|
p(types_js_1.EntityType.GPS_COORDINATES, String.raw `-?(?:[1-8]?\d(?:\.\d{4,})|90(?:\.0{4,}))\s*[,;/]\s*-?(?:1[0-7]\d(?:\.\d{4,})|180(?:\.0{4,})|\d{1,2}(?:\.\d{4,}))`),
|
|
50
50
|
p(types_js_1.EntityType.UUID, String.raw `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b`),
|
|
51
51
|
p(types_js_1.EntityType.SOCIAL_HANDLE, String.raw `(?<!\w)@[a-zA-Z][a-zA-Z0-9_.]{1,29}\b`),
|
|
52
|
+
// --- Secret / API Key (known prefixes — always active) ---
|
|
53
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bAKIA[A-Z0-9]{16}\b`, null, "AWS Access Key ID"),
|
|
54
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bgh[poas]_[a-zA-Z0-9]{36,}\b`, null, "GitHub token"),
|
|
55
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bgithub_pat_[a-zA-Z0-9_]{22,}\b`, null, "GitHub fine-grained PAT"),
|
|
56
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bgl[a-z]t-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab token"),
|
|
57
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bglpat-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab PAT"),
|
|
58
|
+
p(types_js_1.EntityType.SECRET, String.raw `\b[sp]k_(?:live|test)_[a-zA-Z0-9]{24,}\b`, null, "Stripe key"),
|
|
59
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bsk-(?:ant-)?[a-zA-Z0-9\-_]{20,}\b`, null, "OpenAI/Anthropic key"),
|
|
60
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bxox[bpas]-\d[\d\-]{10,}`, null, "Slack numeric token"),
|
|
61
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bxox[bpas]-[a-zA-Z0-9\-]{10,}`, null, "Slack token"),
|
|
62
|
+
p(types_js_1.EntityType.SECRET, String.raw `\beyJ[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\b`, null, "JWT token"),
|
|
63
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bSG\.[a-zA-Z0-9_\-]{22}\.[a-zA-Z0-9_\-]{43}\b`, null, "SendGrid key"),
|
|
64
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bhvs\.[a-zA-Z0-9_\-]{20,}\b`, null, "Vault token"),
|
|
65
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bkey-[a-f0-9]{20,}\b`, null, "Mailgun key"),
|
|
66
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bnpm_[a-zA-Z0-9]{36,}\b`, null, "npm token"),
|
|
67
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bAIza[a-zA-Z0-9_\-]{35}\b`, null, "Google API key"),
|
|
68
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bdop_v1_[a-f0-9]{64}\b`, null, "DigitalOcean token"),
|
|
69
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bAC[a-f0-9]{32}\b`, null, "Twilio Account SID"),
|
|
70
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bxapp-\d[\d\-]{20,}`, null, "Slack app numeric token"),
|
|
71
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bxapp-[a-zA-Z0-9\-]{20,}`, null, "Slack app token"),
|
|
72
|
+
p(types_js_1.EntityType.SECRET, String.raw `\bdapi[a-f0-9]{32,}\b`, null, "Databricks PAT"),
|
|
73
|
+
// --- Secret / API Key (connection strings with embedded credentials) ---
|
|
74
|
+
p(types_js_1.EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]+:[^\s@]+@[^\s]+`, null, "Connection string with credentials"),
|
|
75
|
+
// --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
|
|
76
|
+
p(types_js_1.EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", [
|
|
77
|
+
"key", "token", "secret", "password", "credential",
|
|
78
|
+
"api_key", "apikey", "api-key", "auth", "bearer",
|
|
79
|
+
"aws_secret_access_key", "access_key", "private_key",
|
|
80
|
+
"wachtwoord", "mot de passe", "Passwort", "Schlüssel",
|
|
81
|
+
"clé", "sleutel", "lösenord", "hasło", "heslo",
|
|
82
|
+
"jelszó", "contraseña", "senha", "parola",
|
|
83
|
+
"AES_KEY", "ENCRYPTION_KEY", "SIGNING_KEY",
|
|
84
|
+
"ACCOUNT_SID", "AUTH_TOKEN", "TWILIO",
|
|
85
|
+
"RAILS_SECRET", "SECRET_KEY_BASE", "DATABASE_URL",
|
|
86
|
+
"NPM_TOKEN", "DIGITALOCEAN", "GOOGLE_API",
|
|
87
|
+
"kľúč", "nøkkel", "nyckel", "avain", "lykill",
|
|
88
|
+
"DB_PASS", "DB_PASSWORD", "DATASOURCE_PASSWORD",
|
|
89
|
+
"JWT_SIGNING", "SIGNING_KEY",
|
|
90
|
+
"passwd", "Kennwort",
|
|
91
|
+
], true),
|
|
92
|
+
// --- Secret / API Key (entropy-based fallback for longer tokens) ---
|
|
93
|
+
p(types_js_1.EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", [
|
|
94
|
+
"key", "token", "secret", "password", "credential",
|
|
95
|
+
"api_key", "apikey", "api-key", "auth", "bearer",
|
|
96
|
+
"aws_secret_access_key", "access_key", "private_key",
|
|
97
|
+
"wachtwoord", "mot de passe", "Passwort", "Schlüssel",
|
|
98
|
+
"clé", "sleutel", "lösenord", "hasło", "heslo",
|
|
99
|
+
"jelszó", "contraseña", "senha", "parola",
|
|
100
|
+
"kľúč", "nøkkel", "nyckel", "avain", "lykill",
|
|
101
|
+
], true),
|
|
52
102
|
p(types_js_1.EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
|
|
53
103
|
p(types_js_1.EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
|
|
54
104
|
p(types_js_1.EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),
|
package/dist/cjs/rules/engine.js
CHANGED
|
@@ -9,6 +9,14 @@ const index_js_1 = require("./countries/index.js");
|
|
|
9
9
|
class RuleEngine {
|
|
10
10
|
matcher = new matchers_js_1.MultiPatternMatcher();
|
|
11
11
|
loadedCountries = new Set();
|
|
12
|
+
addCustomPattern(name, pattern) {
|
|
13
|
+
const pdef = {
|
|
14
|
+
entityType: name, pattern, validator: null,
|
|
15
|
+
description: "", contextKeywords: [], requiresContext: false,
|
|
16
|
+
};
|
|
17
|
+
this.matcher.addPattern(pdef, "CUSTOM");
|
|
18
|
+
this.matcher.compile();
|
|
19
|
+
}
|
|
12
20
|
loadCountries(countryCodes) {
|
|
13
21
|
let configs;
|
|
14
22
|
if (countryCodes === null) {
|
|
@@ -21,7 +29,6 @@ class RuleEngine {
|
|
|
21
29
|
const config = index_js_1.COUNTRY_CONFIGS[code];
|
|
22
30
|
if (config)
|
|
23
31
|
configs.push(config);
|
|
24
|
-
// Silently skip unknown country codes (e.g. "INTL")
|
|
25
32
|
}
|
|
26
33
|
const shared = index_js_1.COUNTRY_CONFIGS["SHARED"];
|
|
27
34
|
if (shared)
|
|
@@ -43,32 +50,66 @@ class RuleEngine {
|
|
|
43
50
|
let rawMatches = this.matcher.scan(text);
|
|
44
51
|
if (countryCodes !== null) {
|
|
45
52
|
const codesUpper = new Set(countryCodes.map(c => c.toUpperCase()));
|
|
46
|
-
rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED");
|
|
53
|
+
rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED" || m.countryCode === "CUSTOM");
|
|
54
|
+
}
|
|
55
|
+
// Matches that have a validator but fail it create suppression zones:
|
|
56
|
+
// the span is recognisably a specific entity (e.g. IBAN-shaped) so
|
|
57
|
+
// overlapping regex-only matches (license plate, phone) are false
|
|
58
|
+
// positives and must be suppressed.
|
|
59
|
+
const validated = [];
|
|
60
|
+
const suppressionZones = [];
|
|
61
|
+
for (const m of rawMatches) {
|
|
62
|
+
if (this.matcher.validate(m)) {
|
|
63
|
+
validated.push(m);
|
|
64
|
+
}
|
|
65
|
+
else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
|
|
66
|
+
suppressionZones.push([m.start, m.end]);
|
|
67
|
+
}
|
|
47
68
|
}
|
|
48
|
-
|
|
49
|
-
const
|
|
69
|
+
// Build candidates with priority: validated (3) > custom (2) > regex-only (1)
|
|
70
|
+
const candidates = [];
|
|
50
71
|
for (const match of validated) {
|
|
51
72
|
if ((0, suppressors_js_1.shouldSuppress)(text, match))
|
|
52
73
|
continue;
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
74
|
+
if (match.patternDef.validator === null && suppressionZones.length > 0) {
|
|
75
|
+
if (suppressionZones.some(([zStart, zEnd]) => match.start >= zStart && match.end <= zEnd)) {
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const hasValidator = match.patternDef.validator !== null;
|
|
80
|
+
const isValid = this.matcher.validate(match);
|
|
81
|
+
const priority = (hasValidator && isValid) ? 3 : match.countryCode === "CUSTOM" ? 2 : 1;
|
|
82
|
+
const spanLength = match.end - match.start;
|
|
83
|
+
const score = priority * 1_000_000 + spanLength;
|
|
84
|
+
candidates.push({
|
|
85
|
+
det: {
|
|
86
|
+
entityType: match.patternDef.entityType,
|
|
87
|
+
start: match.start,
|
|
88
|
+
end: match.end,
|
|
89
|
+
text: match.text,
|
|
90
|
+
source: types_js_1.DetectionSource.RULES,
|
|
91
|
+
country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM") ? match.countryCode : null,
|
|
92
|
+
confidence: "high",
|
|
93
|
+
},
|
|
94
|
+
score,
|
|
61
95
|
});
|
|
62
96
|
}
|
|
63
|
-
|
|
64
|
-
|
|
97
|
+
for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
|
|
98
|
+
candidates.push({ det: d, score: 1_000_000 + (d.end - d.start) });
|
|
99
|
+
}
|
|
100
|
+
return this.deduplicate(candidates);
|
|
65
101
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
102
|
+
/**
|
|
103
|
+
* Remove overlapping detections with priority-aware resolution.
|
|
104
|
+
* Priority: validated (3) > custom (2) > regex-only (1).
|
|
105
|
+
* Within the same tier, longer span wins.
|
|
106
|
+
*/
|
|
107
|
+
deduplicate(candidates) {
|
|
108
|
+
if (candidates.length === 0)
|
|
109
|
+
return [];
|
|
110
|
+
const sorted = [...candidates].sort((a, b) => b.score - a.score);
|
|
70
111
|
const result = [];
|
|
71
|
-
for (const det of sorted) {
|
|
112
|
+
for (const { det } of sorted) {
|
|
72
113
|
let overlaps = false;
|
|
73
114
|
for (const kept of result) {
|
|
74
115
|
if (det.start < kept.end && det.end > kept.start) {
|
|
@@ -5,6 +5,11 @@ const validators_js_1 = require("./validators.js");
|
|
|
5
5
|
class MultiPatternMatcher {
|
|
6
6
|
patterns = [];
|
|
7
7
|
compiled = false;
|
|
8
|
+
addPattern(patternDef, countryCode) {
|
|
9
|
+
const regex = new RegExp(patternDef.pattern, "gu");
|
|
10
|
+
this.patterns.push({ regex, patternDef, countryCode });
|
|
11
|
+
this.compiled = false;
|
|
12
|
+
}
|
|
8
13
|
addCountry(config) {
|
|
9
14
|
for (const pdef of config.patterns) {
|
|
10
15
|
// Convert Python-style \b word boundaries — JS handles them natively
|
|
@@ -623,6 +623,35 @@ function validateGreekAmka(candidate) {
|
|
|
623
623
|
}
|
|
624
624
|
return total % 10 === 0;
|
|
625
625
|
}
|
|
626
|
+
function validateHighEntropy(candidate) {
|
|
627
|
+
const clean = candidate.trim();
|
|
628
|
+
if (clean.length < 8)
|
|
629
|
+
return false;
|
|
630
|
+
// Short passwords (< 24 chars): require at least 1 letter, 1 digit, 1 special char
|
|
631
|
+
if (clean.length < 24) {
|
|
632
|
+
const hasLetter = /[a-zA-Z]/.test(clean);
|
|
633
|
+
const hasDigit = /\d/.test(clean);
|
|
634
|
+
const hasSpecial = /[^a-zA-Z0-9]/.test(clean);
|
|
635
|
+
if (hasLetter && hasDigit && hasSpecial)
|
|
636
|
+
return true;
|
|
637
|
+
// For short tokens without special chars, require entropy check
|
|
638
|
+
if (!hasSpecial && clean.length < 16)
|
|
639
|
+
return false;
|
|
640
|
+
}
|
|
641
|
+
const freq = new Map();
|
|
642
|
+
for (const ch of clean)
|
|
643
|
+
freq.set(ch, (freq.get(ch) ?? 0) + 1);
|
|
644
|
+
const len = clean.length;
|
|
645
|
+
let entropy = 0;
|
|
646
|
+
for (const c of freq.values()) {
|
|
647
|
+
const p = c / len;
|
|
648
|
+
entropy -= p * Math.log2(p);
|
|
649
|
+
}
|
|
650
|
+
// Hex-only strings have max ~4.0 bits entropy, use lower threshold
|
|
651
|
+
const isHex = /^[a-fA-F0-9]+$/.test(clean);
|
|
652
|
+
const threshold = isHex ? 3.0 : clean.length < 24 ? 3.0 : 3.5;
|
|
653
|
+
return entropy >= threshold;
|
|
654
|
+
}
|
|
626
655
|
exports.VALIDATORS = {
|
|
627
656
|
iban: validateIban,
|
|
628
657
|
bsn: validateBsn,
|
|
@@ -666,4 +695,5 @@ exports.VALIDATORS = {
|
|
|
666
695
|
uk_nhs: validateUkNhs,
|
|
667
696
|
greek_afm: validateGreekAfm,
|
|
668
697
|
greek_amka: validateGreekAmka,
|
|
698
|
+
high_entropy: validateHighEntropy,
|
|
669
699
|
};
|
package/dist/cjs/sdk.js
CHANGED
|
@@ -6,10 +6,10 @@ const normalizer_js_1 = require("./normalizer.js");
|
|
|
6
6
|
const engine_js_1 = require("./rules/engine.js");
|
|
7
7
|
const types_js_1 = require("./types.js");
|
|
8
8
|
const DATE_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
|
|
9
|
-
class
|
|
9
|
+
class ReferentialMapper {
|
|
10
10
|
counters = new Map();
|
|
11
11
|
mapping = new Map();
|
|
12
|
-
|
|
12
|
+
getLabel(text, entityType) {
|
|
13
13
|
if (!this.mapping.has(text)) {
|
|
14
14
|
const count = (this.counters.get(entityType) ?? 0) + 1;
|
|
15
15
|
this.counters.set(entityType, count);
|
|
@@ -21,9 +21,13 @@ class PseudonymMapper {
|
|
|
21
21
|
class EuRedact {
|
|
22
22
|
engine = new engine_js_1.RuleEngine();
|
|
23
23
|
cache = new cache_js_1.ResultCache();
|
|
24
|
-
|
|
24
|
+
referentialMapper = new ReferentialMapper();
|
|
25
|
+
addCustomPattern(name, pattern) {
|
|
26
|
+
this.engine.addCustomPattern(name, pattern);
|
|
27
|
+
this.cache.clear();
|
|
28
|
+
}
|
|
25
29
|
redact(text, options = {}) {
|
|
26
|
-
const { countries = null, mode = "rules",
|
|
30
|
+
const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
|
|
27
31
|
const [normalizedText, offsetMapping] = (0, normalizer_js_1.normalize)(text);
|
|
28
32
|
const countriesTuple = countries
|
|
29
33
|
? countries.map(c => c.toUpperCase()).sort()
|
|
@@ -51,8 +55,8 @@ class EuRedact {
|
|
|
51
55
|
let redacted = text;
|
|
52
56
|
for (let i = detections.length - 1; i >= 0; i--) {
|
|
53
57
|
const det = detections[i];
|
|
54
|
-
const replacement =
|
|
55
|
-
? this.
|
|
58
|
+
const replacement = referentialIntegrity
|
|
59
|
+
? this.referentialMapper.getLabel(det.text, det.entityType)
|
|
56
60
|
: `[${det.entityType}]`;
|
|
57
61
|
redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
|
|
58
62
|
}
|
package/dist/cjs/types.js
CHANGED
|
@@ -32,6 +32,7 @@ var EntityType;
|
|
|
32
32
|
EntityType["GPS_COORDINATES"] = "GPS_COORDINATES";
|
|
33
33
|
EntityType["UUID"] = "UUID";
|
|
34
34
|
EntityType["SOCIAL_HANDLE"] = "SOCIAL_HANDLE";
|
|
35
|
+
EntityType["SECRET"] = "SECRET";
|
|
35
36
|
EntityType["OTHER"] = "OTHER";
|
|
36
37
|
})(EntityType || (exports.EntityType = EntityType = {}));
|
|
37
38
|
var DetectionSource;
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -6,4 +6,5 @@ import { type RedactOptions } from "./sdk.js";
|
|
|
6
6
|
import type { RedactResult } from "./types.js";
|
|
7
7
|
export declare function availableCountries(): string[];
|
|
8
8
|
export declare function redact(text: string, options?: RedactOptions): RedactResult;
|
|
9
|
+
export declare function addCustomPattern(name: string, pattern: string): void;
|
|
9
10
|
export declare function redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
|
package/dist/esm/index.js
CHANGED
|
@@ -15,6 +15,9 @@ export function availableCountries() {
|
|
|
15
15
|
export function redact(text, options) {
|
|
16
16
|
return getInstance().redact(text, options);
|
|
17
17
|
}
|
|
18
|
+
export function addCustomPattern(name, pattern) {
|
|
19
|
+
getInstance().addCustomPattern(name, pattern);
|
|
20
|
+
}
|
|
18
21
|
export function redactBatch(texts, options) {
|
|
19
22
|
return getInstance().redactBatch(texts, options);
|
|
20
23
|
}
|
|
@@ -46,6 +46,56 @@ const SHARED = {
|
|
|
46
46
|
p(EntityType.GPS_COORDINATES, String.raw `-?(?:[1-8]?\d(?:\.\d{4,})|90(?:\.0{4,}))\s*[,;/]\s*-?(?:1[0-7]\d(?:\.\d{4,})|180(?:\.0{4,})|\d{1,2}(?:\.\d{4,}))`),
|
|
47
47
|
p(EntityType.UUID, String.raw `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b`),
|
|
48
48
|
p(EntityType.SOCIAL_HANDLE, String.raw `(?<!\w)@[a-zA-Z][a-zA-Z0-9_.]{1,29}\b`),
|
|
49
|
+
// --- Secret / API Key (known prefixes — always active) ---
|
|
50
|
+
p(EntityType.SECRET, String.raw `\bAKIA[A-Z0-9]{16}\b`, null, "AWS Access Key ID"),
|
|
51
|
+
p(EntityType.SECRET, String.raw `\bgh[poas]_[a-zA-Z0-9]{36,}\b`, null, "GitHub token"),
|
|
52
|
+
p(EntityType.SECRET, String.raw `\bgithub_pat_[a-zA-Z0-9_]{22,}\b`, null, "GitHub fine-grained PAT"),
|
|
53
|
+
p(EntityType.SECRET, String.raw `\bgl[a-z]t-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab token"),
|
|
54
|
+
p(EntityType.SECRET, String.raw `\bglpat-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab PAT"),
|
|
55
|
+
p(EntityType.SECRET, String.raw `\b[sp]k_(?:live|test)_[a-zA-Z0-9]{24,}\b`, null, "Stripe key"),
|
|
56
|
+
p(EntityType.SECRET, String.raw `\bsk-(?:ant-)?[a-zA-Z0-9\-_]{20,}\b`, null, "OpenAI/Anthropic key"),
|
|
57
|
+
p(EntityType.SECRET, String.raw `\bxox[bpas]-\d[\d\-]{10,}`, null, "Slack numeric token"),
|
|
58
|
+
p(EntityType.SECRET, String.raw `\bxox[bpas]-[a-zA-Z0-9\-]{10,}`, null, "Slack token"),
|
|
59
|
+
p(EntityType.SECRET, String.raw `\beyJ[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\b`, null, "JWT token"),
|
|
60
|
+
p(EntityType.SECRET, String.raw `\bSG\.[a-zA-Z0-9_\-]{22}\.[a-zA-Z0-9_\-]{43}\b`, null, "SendGrid key"),
|
|
61
|
+
p(EntityType.SECRET, String.raw `\bhvs\.[a-zA-Z0-9_\-]{20,}\b`, null, "Vault token"),
|
|
62
|
+
p(EntityType.SECRET, String.raw `\bkey-[a-f0-9]{20,}\b`, null, "Mailgun key"),
|
|
63
|
+
p(EntityType.SECRET, String.raw `\bnpm_[a-zA-Z0-9]{36,}\b`, null, "npm token"),
|
|
64
|
+
p(EntityType.SECRET, String.raw `\bAIza[a-zA-Z0-9_\-]{35}\b`, null, "Google API key"),
|
|
65
|
+
p(EntityType.SECRET, String.raw `\bdop_v1_[a-f0-9]{64}\b`, null, "DigitalOcean token"),
|
|
66
|
+
p(EntityType.SECRET, String.raw `\bAC[a-f0-9]{32}\b`, null, "Twilio Account SID"),
|
|
67
|
+
p(EntityType.SECRET, String.raw `\bxapp-\d[\d\-]{20,}`, null, "Slack app numeric token"),
|
|
68
|
+
p(EntityType.SECRET, String.raw `\bxapp-[a-zA-Z0-9\-]{20,}`, null, "Slack app token"),
|
|
69
|
+
p(EntityType.SECRET, String.raw `\bdapi[a-f0-9]{32,}\b`, null, "Databricks PAT"),
|
|
70
|
+
// --- Secret / API Key (connection strings with embedded credentials) ---
|
|
71
|
+
p(EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]+:[^\s@]+@[^\s]+`, null, "Connection string with credentials"),
|
|
72
|
+
// --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
|
|
73
|
+
p(EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", [
|
|
74
|
+
"key", "token", "secret", "password", "credential",
|
|
75
|
+
"api_key", "apikey", "api-key", "auth", "bearer",
|
|
76
|
+
"aws_secret_access_key", "access_key", "private_key",
|
|
77
|
+
"wachtwoord", "mot de passe", "Passwort", "Schlüssel",
|
|
78
|
+
"clé", "sleutel", "lösenord", "hasło", "heslo",
|
|
79
|
+
"jelszó", "contraseña", "senha", "parola",
|
|
80
|
+
"AES_KEY", "ENCRYPTION_KEY", "SIGNING_KEY",
|
|
81
|
+
"ACCOUNT_SID", "AUTH_TOKEN", "TWILIO",
|
|
82
|
+
"RAILS_SECRET", "SECRET_KEY_BASE", "DATABASE_URL",
|
|
83
|
+
"NPM_TOKEN", "DIGITALOCEAN", "GOOGLE_API",
|
|
84
|
+
"kľúč", "nøkkel", "nyckel", "avain", "lykill",
|
|
85
|
+
"DB_PASS", "DB_PASSWORD", "DATASOURCE_PASSWORD",
|
|
86
|
+
"JWT_SIGNING", "SIGNING_KEY",
|
|
87
|
+
"passwd", "Kennwort",
|
|
88
|
+
], true),
|
|
89
|
+
// --- Secret / API Key (entropy-based fallback for longer tokens) ---
|
|
90
|
+
p(EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", [
|
|
91
|
+
"key", "token", "secret", "password", "credential",
|
|
92
|
+
"api_key", "apikey", "api-key", "auth", "bearer",
|
|
93
|
+
"aws_secret_access_key", "access_key", "private_key",
|
|
94
|
+
"wachtwoord", "mot de passe", "Passwort", "Schlüssel",
|
|
95
|
+
"clé", "sleutel", "lösenord", "hasło", "heslo",
|
|
96
|
+
"jelszó", "contraseña", "senha", "parola",
|
|
97
|
+
"kľúč", "nøkkel", "nyckel", "avain", "lykill",
|
|
98
|
+
], true),
|
|
49
99
|
p(EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
|
|
50
100
|
p(EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
|
|
51
101
|
p(EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),
|
|
@@ -2,8 +2,14 @@ import { type Detection } from "../types.js";
|
|
|
2
2
|
export declare class RuleEngine {
|
|
3
3
|
private matcher;
|
|
4
4
|
private loadedCountries;
|
|
5
|
+
addCustomPattern(name: string, pattern: string): void;
|
|
5
6
|
loadCountries(countryCodes: string[] | null): void;
|
|
6
7
|
detect(text: string, countryCodes: string[] | null): Detection[];
|
|
8
|
+
/**
|
|
9
|
+
* Remove overlapping detections with priority-aware resolution.
|
|
10
|
+
* Priority: validated (3) > custom (2) > regex-only (1).
|
|
11
|
+
* Within the same tier, longer span wins.
|
|
12
|
+
*/
|
|
7
13
|
private deduplicate;
|
|
8
14
|
getLoadedCountries(): Set<string>;
|
|
9
15
|
}
|
package/dist/esm/rules/engine.js
CHANGED
|
@@ -6,6 +6,14 @@ import { COUNTRY_CONFIGS } from "./countries/index.js";
|
|
|
6
6
|
export class RuleEngine {
|
|
7
7
|
matcher = new MultiPatternMatcher();
|
|
8
8
|
loadedCountries = new Set();
|
|
9
|
+
addCustomPattern(name, pattern) {
|
|
10
|
+
const pdef = {
|
|
11
|
+
entityType: name, pattern, validator: null,
|
|
12
|
+
description: "", contextKeywords: [], requiresContext: false,
|
|
13
|
+
};
|
|
14
|
+
this.matcher.addPattern(pdef, "CUSTOM");
|
|
15
|
+
this.matcher.compile();
|
|
16
|
+
}
|
|
9
17
|
loadCountries(countryCodes) {
|
|
10
18
|
let configs;
|
|
11
19
|
if (countryCodes === null) {
|
|
@@ -18,7 +26,6 @@ export class RuleEngine {
|
|
|
18
26
|
const config = COUNTRY_CONFIGS[code];
|
|
19
27
|
if (config)
|
|
20
28
|
configs.push(config);
|
|
21
|
-
// Silently skip unknown country codes (e.g. "INTL")
|
|
22
29
|
}
|
|
23
30
|
const shared = COUNTRY_CONFIGS["SHARED"];
|
|
24
31
|
if (shared)
|
|
@@ -40,32 +47,66 @@ export class RuleEngine {
|
|
|
40
47
|
let rawMatches = this.matcher.scan(text);
|
|
41
48
|
if (countryCodes !== null) {
|
|
42
49
|
const codesUpper = new Set(countryCodes.map(c => c.toUpperCase()));
|
|
43
|
-
rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED");
|
|
50
|
+
rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED" || m.countryCode === "CUSTOM");
|
|
51
|
+
}
|
|
52
|
+
// Matches that have a validator but fail it create suppression zones:
|
|
53
|
+
// the span is recognisably a specific entity (e.g. IBAN-shaped) so
|
|
54
|
+
// overlapping regex-only matches (license plate, phone) are false
|
|
55
|
+
// positives and must be suppressed.
|
|
56
|
+
const validated = [];
|
|
57
|
+
const suppressionZones = [];
|
|
58
|
+
for (const m of rawMatches) {
|
|
59
|
+
if (this.matcher.validate(m)) {
|
|
60
|
+
validated.push(m);
|
|
61
|
+
}
|
|
62
|
+
else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
|
|
63
|
+
suppressionZones.push([m.start, m.end]);
|
|
64
|
+
}
|
|
44
65
|
}
|
|
45
|
-
|
|
46
|
-
const
|
|
66
|
+
// Build candidates with priority: validated (3) > custom (2) > regex-only (1)
|
|
67
|
+
const candidates = [];
|
|
47
68
|
for (const match of validated) {
|
|
48
69
|
if (shouldSuppress(text, match))
|
|
49
70
|
continue;
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
71
|
+
if (match.patternDef.validator === null && suppressionZones.length > 0) {
|
|
72
|
+
if (suppressionZones.some(([zStart, zEnd]) => match.start >= zStart && match.end <= zEnd)) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const hasValidator = match.patternDef.validator !== null;
|
|
77
|
+
const isValid = this.matcher.validate(match);
|
|
78
|
+
const priority = (hasValidator && isValid) ? 3 : match.countryCode === "CUSTOM" ? 2 : 1;
|
|
79
|
+
const spanLength = match.end - match.start;
|
|
80
|
+
const score = priority * 1_000_000 + spanLength;
|
|
81
|
+
candidates.push({
|
|
82
|
+
det: {
|
|
83
|
+
entityType: match.patternDef.entityType,
|
|
84
|
+
start: match.start,
|
|
85
|
+
end: match.end,
|
|
86
|
+
text: match.text,
|
|
87
|
+
source: DetectionSource.RULES,
|
|
88
|
+
country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM") ? match.countryCode : null,
|
|
89
|
+
confidence: "high",
|
|
90
|
+
},
|
|
91
|
+
score,
|
|
58
92
|
});
|
|
59
93
|
}
|
|
60
|
-
|
|
61
|
-
|
|
94
|
+
for (const d of detectStructuralDob(text)) {
|
|
95
|
+
candidates.push({ det: d, score: 1_000_000 + (d.end - d.start) });
|
|
96
|
+
}
|
|
97
|
+
return this.deduplicate(candidates);
|
|
62
98
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
99
|
+
/**
|
|
100
|
+
* Remove overlapping detections with priority-aware resolution.
|
|
101
|
+
* Priority: validated (3) > custom (2) > regex-only (1).
|
|
102
|
+
* Within the same tier, longer span wins.
|
|
103
|
+
*/
|
|
104
|
+
deduplicate(candidates) {
|
|
105
|
+
if (candidates.length === 0)
|
|
106
|
+
return [];
|
|
107
|
+
const sorted = [...candidates].sort((a, b) => b.score - a.score);
|
|
67
108
|
const result = [];
|
|
68
|
-
for (const det of sorted) {
|
|
109
|
+
for (const { det } of sorted) {
|
|
69
110
|
let overlaps = false;
|
|
70
111
|
for (const kept of result) {
|
|
71
112
|
if (det.start < kept.end && det.end > kept.start) {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import type { CountryConfig } from "../types.js";
|
|
1
|
+
import type { CountryConfig, PatternDef } from "../types.js";
|
|
2
2
|
import type { RawMatch } from "./suppressors.js";
|
|
3
3
|
export type { RawMatch };
|
|
4
4
|
export declare class MultiPatternMatcher {
|
|
5
5
|
private patterns;
|
|
6
6
|
private compiled;
|
|
7
|
+
addPattern(patternDef: PatternDef, countryCode: string): void;
|
|
7
8
|
addCountry(config: CountryConfig): void;
|
|
8
9
|
compile(): void;
|
|
9
10
|
scan(text: string): RawMatch[];
|
|
@@ -2,6 +2,11 @@ import { VALIDATORS } from "./validators.js";
|
|
|
2
2
|
export class MultiPatternMatcher {
|
|
3
3
|
patterns = [];
|
|
4
4
|
compiled = false;
|
|
5
|
+
addPattern(patternDef, countryCode) {
|
|
6
|
+
const regex = new RegExp(patternDef.pattern, "gu");
|
|
7
|
+
this.patterns.push({ regex, patternDef, countryCode });
|
|
8
|
+
this.compiled = false;
|
|
9
|
+
}
|
|
5
10
|
addCountry(config) {
|
|
6
11
|
for (const pdef of config.patterns) {
|
|
7
12
|
// Convert Python-style \b word boundaries — JS handles them natively
|
|
@@ -578,6 +578,35 @@ export function validateGreekAmka(candidate) {
|
|
|
578
578
|
}
|
|
579
579
|
return total % 10 === 0;
|
|
580
580
|
}
|
|
581
|
+
function validateHighEntropy(candidate) {
|
|
582
|
+
const clean = candidate.trim();
|
|
583
|
+
if (clean.length < 8)
|
|
584
|
+
return false;
|
|
585
|
+
// Short passwords (< 24 chars): require at least 1 letter, 1 digit, 1 special char
|
|
586
|
+
if (clean.length < 24) {
|
|
587
|
+
const hasLetter = /[a-zA-Z]/.test(clean);
|
|
588
|
+
const hasDigit = /\d/.test(clean);
|
|
589
|
+
const hasSpecial = /[^a-zA-Z0-9]/.test(clean);
|
|
590
|
+
if (hasLetter && hasDigit && hasSpecial)
|
|
591
|
+
return true;
|
|
592
|
+
// For short tokens without special chars, require entropy check
|
|
593
|
+
if (!hasSpecial && clean.length < 16)
|
|
594
|
+
return false;
|
|
595
|
+
}
|
|
596
|
+
const freq = new Map();
|
|
597
|
+
for (const ch of clean)
|
|
598
|
+
freq.set(ch, (freq.get(ch) ?? 0) + 1);
|
|
599
|
+
const len = clean.length;
|
|
600
|
+
let entropy = 0;
|
|
601
|
+
for (const c of freq.values()) {
|
|
602
|
+
const p = c / len;
|
|
603
|
+
entropy -= p * Math.log2(p);
|
|
604
|
+
}
|
|
605
|
+
// Hex-only strings have max ~4.0 bits entropy, use lower threshold
|
|
606
|
+
const isHex = /^[a-fA-F0-9]+$/.test(clean);
|
|
607
|
+
const threshold = isHex ? 3.0 : clean.length < 24 ? 3.0 : 3.5;
|
|
608
|
+
return entropy >= threshold;
|
|
609
|
+
}
|
|
581
610
|
export const VALIDATORS = {
|
|
582
611
|
iban: validateIban,
|
|
583
612
|
bsn: validateBsn,
|
|
@@ -621,4 +650,5 @@ export const VALIDATORS = {
|
|
|
621
650
|
uk_nhs: validateUkNhs,
|
|
622
651
|
greek_afm: validateGreekAfm,
|
|
623
652
|
greek_amka: validateGreekAmka,
|
|
653
|
+
high_entropy: validateHighEntropy,
|
|
624
654
|
};
|
package/dist/esm/sdk.d.ts
CHANGED
|
@@ -2,14 +2,15 @@ import { type RedactResult } from "./types.js";
|
|
|
2
2
|
export interface RedactOptions {
|
|
3
3
|
countries?: string[] | null;
|
|
4
4
|
mode?: string;
|
|
5
|
-
|
|
5
|
+
referentialIntegrity?: boolean;
|
|
6
6
|
detectDates?: boolean;
|
|
7
7
|
cache?: boolean;
|
|
8
8
|
}
|
|
9
9
|
export declare class EuRedact {
|
|
10
10
|
private engine;
|
|
11
11
|
private cache;
|
|
12
|
-
private
|
|
12
|
+
private referentialMapper;
|
|
13
|
+
addCustomPattern(name: string, pattern: string): void;
|
|
13
14
|
redact(text: string, options?: RedactOptions): RedactResult;
|
|
14
15
|
redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
|
|
15
16
|
}
|
package/dist/esm/sdk.js
CHANGED
|
@@ -3,10 +3,10 @@ import { normalize, mapOffsetToOriginal } from "./normalizer.js";
|
|
|
3
3
|
import { RuleEngine } from "./rules/engine.js";
|
|
4
4
|
import { EntityType } from "./types.js";
|
|
5
5
|
const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
|
|
6
|
-
class
|
|
6
|
+
class ReferentialMapper {
|
|
7
7
|
counters = new Map();
|
|
8
8
|
mapping = new Map();
|
|
9
|
-
|
|
9
|
+
getLabel(text, entityType) {
|
|
10
10
|
if (!this.mapping.has(text)) {
|
|
11
11
|
const count = (this.counters.get(entityType) ?? 0) + 1;
|
|
12
12
|
this.counters.set(entityType, count);
|
|
@@ -18,9 +18,13 @@ class PseudonymMapper {
|
|
|
18
18
|
export class EuRedact {
|
|
19
19
|
engine = new RuleEngine();
|
|
20
20
|
cache = new ResultCache();
|
|
21
|
-
|
|
21
|
+
referentialMapper = new ReferentialMapper();
|
|
22
|
+
addCustomPattern(name, pattern) {
|
|
23
|
+
this.engine.addCustomPattern(name, pattern);
|
|
24
|
+
this.cache.clear();
|
|
25
|
+
}
|
|
22
26
|
redact(text, options = {}) {
|
|
23
|
-
const { countries = null, mode = "rules",
|
|
27
|
+
const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
|
|
24
28
|
const [normalizedText, offsetMapping] = normalize(text);
|
|
25
29
|
const countriesTuple = countries
|
|
26
30
|
? countries.map(c => c.toUpperCase()).sort()
|
|
@@ -48,8 +52,8 @@ export class EuRedact {
|
|
|
48
52
|
let redacted = text;
|
|
49
53
|
for (let i = detections.length - 1; i >= 0; i--) {
|
|
50
54
|
const det = detections[i];
|
|
51
|
-
const replacement =
|
|
52
|
-
? this.
|
|
55
|
+
const replacement = referentialIntegrity
|
|
56
|
+
? this.referentialMapper.getLabel(det.text, det.entityType)
|
|
53
57
|
: `[${det.entityType}]`;
|
|
54
58
|
redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
|
|
55
59
|
}
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -28,6 +28,7 @@ export declare enum EntityType {
|
|
|
28
28
|
GPS_COORDINATES = "GPS_COORDINATES",
|
|
29
29
|
UUID = "UUID",
|
|
30
30
|
SOCIAL_HANDLE = "SOCIAL_HANDLE",
|
|
31
|
+
SECRET = "SECRET",
|
|
31
32
|
OTHER = "OTHER"
|
|
32
33
|
}
|
|
33
34
|
export declare enum DetectionSource {
|
|
@@ -35,7 +36,7 @@ export declare enum DetectionSource {
|
|
|
35
36
|
CLOUD = "cloud"
|
|
36
37
|
}
|
|
37
38
|
export interface Detection {
|
|
38
|
-
entityType: EntityType;
|
|
39
|
+
entityType: EntityType | string;
|
|
39
40
|
start: number;
|
|
40
41
|
end: number;
|
|
41
42
|
text: string;
|
|
@@ -50,7 +51,7 @@ export interface RedactResult {
|
|
|
50
51
|
degraded: boolean;
|
|
51
52
|
}
|
|
52
53
|
export interface PatternDef {
|
|
53
|
-
entityType: EntityType;
|
|
54
|
+
entityType: EntityType | string;
|
|
54
55
|
pattern: string;
|
|
55
56
|
validator: string | null;
|
|
56
57
|
description: string;
|
package/dist/esm/types.js
CHANGED
|
@@ -29,6 +29,7 @@ export var EntityType;
|
|
|
29
29
|
EntityType["GPS_COORDINATES"] = "GPS_COORDINATES";
|
|
30
30
|
EntityType["UUID"] = "UUID";
|
|
31
31
|
EntityType["SOCIAL_HANDLE"] = "SOCIAL_HANDLE";
|
|
32
|
+
EntityType["SECRET"] = "SECRET";
|
|
32
33
|
EntityType["OTHER"] = "OTHER";
|
|
33
34
|
})(EntityType || (EntityType = {}));
|
|
34
35
|
export var DetectionSource;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
],
|
|
47
47
|
"repository": {
|
|
48
48
|
"type": "git",
|
|
49
|
-
"url": "https://github.com/
|
|
49
|
+
"url": "https://github.com/euredact/euredact"
|
|
50
50
|
},
|
|
51
51
|
"scripts": {
|
|
52
52
|
"build:esm": "tsc",
|