euredact 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,8 +1,13 @@
1
1
  # euredact
2
2
 
3
- Fast, zero-dependency European PII detection and redaction for Node.js.
3
+ **European PII detection and redaction for Node.js / TypeScript**
4
4
 
5
- Detects and redacts personal data across **31 European countries** using rule-based pattern matching with checksum validation and context-aware false-positive suppression.
5
+ ---
6
+
7
+ Fast, zero-dependency European PII detection and redaction SDK. Detects and
8
+ redacts personal data across **31 European countries** using rule-based pattern
9
+ matching with checksum validation, context-aware false-positive suppression,
10
+ and priority-based deduplication.
6
11
 
7
12
  ## Install
8
13
 
@@ -26,64 +31,304 @@ console.log(result.detections);
26
31
  // [{ entityType: "NATIONAL_ID", text: "123456782", start: 12, end: 21, ... }, ...]
27
32
  ```
28
33
 
29
- ## API
34
+ ## Features
35
+
36
+ - **31 European countries** with country-specific patterns
37
+ - **25+ PII entity types:** national IDs, IBANs, phone numbers, email, VAT
38
+ numbers, license plates, credit cards, BIC/SWIFT, VIN, IMEI, GPS coordinates,
39
+ UUIDs, social handles, IP/IPv6, MAC addresses, secrets/API keys, and more
40
+ - **Secret/API key detection:** known-prefix patterns for AWS, GitHub, Stripe,
41
+ OpenAI, Slack, JWT, SendGrid, plus Shannon entropy-based detection for generic
42
+ high-entropy tokens near context keywords
43
+ - **Custom patterns:** register your own regex patterns for domain-specific PII
44
+ - **Checksum validation:** IBAN mod-97, Luhn (credit cards/IMEI), and 30+
45
+ country-specific validators
46
+ - **Priority-aware deduplication:** validated (checksum) > custom > regex-only
47
+ - **Context-aware:** keyword proximity checks and structural detection (JSON field
48
+ names, CSV headers) for ambiguous patterns
49
+ - **Zero runtime dependencies**
50
+ - **ESM and CommonJS** dual-published
51
+
52
+ ### Supported Countries
30
53
 
31
- ### `redact(text, options?)`
54
+ | Region | Countries |
55
+ |---|---|
56
+ | Western Europe | AT, BE, CH, DE, FR, LU, NL |
57
+ | Southern Europe | CY, EL, ES, IT, MT, PT |
58
+ | Northern Europe | DK, EE, FI, IS, LT, LV, NO, SE |
59
+ | Eastern Europe | BG, CZ, HR, HU, PL, RO, SI, SK |
60
+ | British Isles | IE, UK |
32
61
 
33
- Returns a `RedactResult` with the redacted text and a list of detections.
62
+ ## API Reference
63
+
64
+ ### Module-Level Functions
65
+
66
+ #### `redact(text, options?)`
67
+
68
+ ```ts
69
+ function redact(text: string, options?: RedactOptions): RedactResult;
70
+ ```
71
+
72
+ Main entry point. Detects and redacts PII in the given text.
34
73
 
35
74
  ```ts
36
75
  interface RedactOptions {
37
- countries?: string[]; // Country codes (e.g. ["NL", "BE"]) — recommended for best precision
38
- pseudonymize?: boolean; // Replace with consistent pseudonyms (EMAIL_1, EMAIL_2, ...)
39
- detectDates?: boolean; // Include DOB/date-of-death detections (off by default)
40
- cache?: boolean; // Enable result caching (default: true)
76
+ countries?: string[] | null; // Country codes (e.g. ["NL", "BE"]) — null loads all
77
+ referentialIntegrity?: boolean; // Replace with consistent labels (default: false)
78
+ detectDates?: boolean; // Include DOB/date-of-death detections (default: false)
79
+ cache?: boolean; // Enable result caching (default: true)
41
80
  }
42
81
  ```
43
82
 
44
- ### `redactBatch(texts, options?)`
83
+ | Parameter | Default | Description |
84
+ |---|---|---|
85
+ | `countries` | `null` | ISO 3166-1 alpha-2 codes to restrict detection. `null` loads all 31 countries. |
86
+ | `referentialIntegrity` | `false` | Replace PII with consistent labels instead of entity-type labels. |
87
+ | `detectDates` | `false` | Include date-of-birth and date-of-death detections. Off by default. |
88
+ | `cache` | `true` | Cache results for identical inputs. |
89
+
90
+ #### `redactBatch(texts, options?)`
91
+
92
+ ```ts
93
+ function redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
94
+ ```
95
+
96
+ Process multiple texts efficiently. Loads country configs once. Returns results
97
+ in the same order as the input.
45
98
 
46
- Process multiple texts efficiently. Loads country configs once.
99
+ #### `addCustomPattern(name, pattern)`
47
100
 
48
- ### `availableCountries()`
101
+ ```ts
102
+ function addCustomPattern(name: string, pattern: string): void;
103
+ ```
49
104
 
50
- Returns the list of supported country codes.
105
+ Register a custom regex pattern. Matches are reported with `name` as the entity
106
+ type. See [Custom Patterns](#custom-patterns) below.
51
107
 
52
- ## Supported Countries
108
+ #### `availableCountries()`
53
109
 
54
- AT, BE, BG, CH, CY, CZ, DE, DK, EE, EL, ES, FI, FR, HR, HU, IE, IS, IT, LT, LU, LV, MT, NL, NO, PL, PT, RO, SE, SI, SK, UK
110
+ ```ts
111
+ function availableCountries(): string[];
112
+ ```
55
113
 
56
- ## Detected PII Types
114
+ Returns a sorted list of supported ISO country codes.
57
115
 
58
- | Category | Examples |
59
- |----------|---------|
60
- | `NATIONAL_ID` | BSN, Personalausweis, NIR, CPR, personnummer, ... |
61
- | `IBAN` | Country-specific IBAN with ISO 13616 checksum |
62
- | `PHONE` | National and international formats per country |
63
- | `EMAIL` | RFC 5322 simplified |
64
- | `POSTAL_CODE` | Country-specific formats with context validation |
65
- | `TAX_ID` | Steuer-ID, NIF, NIP, ... with checksums |
66
- | `CREDIT_CARD` | Visa, Mastercard, Amex with Luhn check |
67
- | `LICENSE_PLATE` | Country-specific formats |
68
- | `PASSPORT` | Context-required per country |
69
- | `VAT` | EU VAT numbers |
70
- | `BIC` | SWIFT/BIC codes |
71
- | `VIN` | ISO 3779 vehicle identification |
72
- | `IP_ADDRESS` | IPv4 and IPv6 |
73
- | `MAC_ADDRESS` | Colon, dash, and Cisco dot notation |
74
- | `UUID` | RFC 4122 versions 1-5 |
75
- | `IMEI` | 15-digit with Luhn check |
76
- | `GPS_COORDINATES` | Decimal degrees with 4+ decimal places |
77
- | `SOCIAL_HANDLE` | @username format |
78
- | `DOB` | Date of birth (context-required, opt-in) |
116
+ ### Instance-Based API (`EuRedact` Class)
79
117
 
80
- ## Performance
118
+ For applications that need isolated instances with separate caches and custom
119
+ pattern registrations:
81
120
 
82
- - **0.02ms** per redaction (typical text)
83
- - **Zero runtime dependencies**
84
- - **86KB** total package size
85
- - Checksum validators eliminate false positives at the pattern level
86
- - Context-aware suppressors filter currency amounts, units, references, and legal citations
121
+ ```ts
122
+ import { EuRedact } from "euredact";
123
+
124
+ const instance = new EuRedact();
125
+ instance.addCustomPattern("CASE_REF", "CASE-\\d{8}");
126
+
127
+ const result = instance.redact("See CASE-20260401 for details", {
128
+ countries: ["NL", "BE"],
129
+ });
130
+ console.log(result.redactedText);
131
+ // "See [CASE_REF] for details"
132
+ ```
133
+
134
+ The `EuRedact` class exposes: `redact()`, `redactBatch()`, and
135
+ `addCustomPattern()`.
136
+
137
+ ### Return Types
138
+
139
+ #### `RedactResult`
140
+
141
+ ```ts
142
+ interface RedactResult {
143
+ redactedText: string; // The input text with PII replaced
144
+ detections: Detection[]; // All PII spans found
145
+ source: string; // Detection backend ("rules")
146
+ degraded: boolean; // True if the engine fell back to a simpler mode
147
+ }
148
+ ```
149
+
150
+ #### `Detection`
151
+
152
+ ```ts
153
+ interface Detection {
154
+ entityType: EntityType | string; // PII category (enum or custom name)
155
+ start: number; // Start offset in the original text
156
+ end: number; // End offset (exclusive)
157
+ text: string; // The matched substring
158
+ source: DetectionSource; // "rules" or "cloud"
159
+ country: string | null; // ISO code or null for shared/custom patterns
160
+ confidence: string; // Confidence level
161
+ }
162
+ ```
163
+
164
+ #### `EntityType`
165
+
166
+ String enum with all supported PII categories:
167
+
168
+ ```
169
+ NAME ADDRESS IBAN BIC
170
+ CREDIT_CARD PHONE EMAIL DOB
171
+ DATE_OF_DEATH NATIONAL_ID SSN TAX_ID
172
+ PASSPORT DRIVERS_LICENSE RESIDENCE_PERMIT LICENSE_PLATE
173
+ VIN VAT POSTAL_CODE IP_ADDRESS
174
+ IPV6_ADDRESS MAC_ADDRESS HEALTH_INSURANCE HEALTHCARE_PROVIDER
175
+ CHAMBER_OF_COMMERCE IMEI GPS_COORDINATES UUID
176
+ SOCIAL_HANDLE SECRET OTHER
177
+ ```
178
+
179
+ For custom patterns, `entityType` is a plain string (e.g. `"EMPLOYEE_ID"`).
180
+
181
+ ## Custom Patterns
182
+
183
+ Register domain-specific PII patterns at runtime:
184
+
185
+ ```ts
186
+ import { addCustomPattern, redact } from "euredact";
187
+
188
+ // Register patterns
189
+ addCustomPattern("EMPLOYEE_ID", "EMP-\\d{6}");
190
+ addCustomPattern("CASE_REF", "CASE-\\d{8}");
191
+
192
+ // They are detected alongside built-in PII
193
+ const result = redact(
194
+ "Employee EMP-123456, email jan@example.com, ref CASE-20260401"
195
+ );
196
+ console.log(result.redactedText);
197
+ // "Employee [EMPLOYEE_ID], email [EMAIL], ref [CASE_REF]"
198
+ ```
199
+
200
+ ### How Custom Patterns Work
201
+
202
+ - `name` becomes the entity type in detections and replacement tags
203
+ - `pattern` is a JavaScript regular expression (same syntax as `RegExp`)
204
+ - Custom patterns are always active regardless of the `countries` option
205
+ - In overlap resolution, custom patterns have higher priority than built-in
206
+ regex-only patterns but lower priority than built-in patterns with a passing
207
+ checksum validator
208
+
209
+ ### Instance Isolation
210
+
211
+ Custom patterns registered on the module-level function apply to the shared
212
+ singleton. For isolated registrations, use separate `EuRedact` instances:
213
+
214
+ ```ts
215
+ import { EuRedact } from "euredact";
216
+
217
+ const a = new EuRedact();
218
+ a.addCustomPattern("EMPLOYEE_ID", "EMP-\\d{6}");
219
+
220
+ const b = new EuRedact();
221
+ b.addCustomPattern("CASE_REF", "CASE-\\d{8}");
222
+ ```
223
+
224
+ ## Secret and API Key Detection
225
+
226
+ Built-in detection for API keys, tokens, and passwords. Always active -- no
227
+ configuration required.
228
+
229
+ ### Known-Prefix Patterns
230
+
231
+ | Pattern | Description |
232
+ |---|---|
233
+ | `AKIA...` | AWS Access Key ID |
234
+ | `ghp_`, `gho_`, `ghs_`, `github_pat_` | GitHub tokens (PAT, OAuth, app, server) |
235
+ | `sk_live_`, `pk_live_`, `sk_test_`, `pk_test_` | Stripe secret and publishable keys |
236
+ | `sk-`, `sk-ant-` | OpenAI and Anthropic API keys |
237
+ | `xoxb-`, `xoxp-`, `xoxa-`, `xoxs-` | Slack tokens |
238
+ | `eyJ...` (3-part base64url) | JWT tokens |
239
+ | `SG.` | SendGrid API keys |
240
+
241
+ ```ts
242
+ const result = redact("My API key is sk-proj-abc123def456ghi789jkl0");
243
+ console.log(result.redactedText);
244
+ // "My API key is [SECRET]"
245
+ ```
246
+
247
+ ### Entropy-Based Detection
248
+
249
+ High-entropy strings (32+ alphanumeric characters) near context keywords like
250
+ `key`, `token`, `secret`, `password`, `credential`, `auth`, or `bearer`
251
+ (including 12 European language translations) are flagged via Shannon entropy
252
+ analysis:
253
+
254
+ ```ts
255
+ redact("The api_key is xK9mPqR7vLnW2bFjY8cGhT4sDfAeU6iO").redactedText;
256
+ // "The api_key is [SECRET]"
257
+
258
+ // Without a context keyword -- not flagged:
259
+ redact("identifier: xK9mPqR7vLnW2bFjY8cGhT4sDfAeU6iO").redactedText;
260
+ // "identifier: xK9mPqR7vLnW2bFjY8cGhT4sDfAeU6iO"
261
+ ```
262
+
263
+ ## Country Hints
264
+
265
+ When you know which countries appear in your data, pass them explicitly:
266
+
267
+ ```ts
268
+ const result = redact(text, { countries: ["NL", "BE"] });
269
+ ```
270
+
271
+ This restricts pattern matching to Dutch and Belgian rules (plus shared patterns
272
+ and custom patterns). Benefits:
273
+
274
+ - **Fewer false positives.** A 9-digit number that passes the Dutch BSN checksum
275
+ won't also be tested against unrelated country patterns.
276
+ - **Faster.** Fewer patterns to compile and scan.
277
+
278
+ ## Referential Integrity
279
+
280
+ When `referentialIntegrity: true`, each unique PII value is mapped to a consistent
281
+ label:
282
+
283
+ ```ts
284
+ const result = redact(
285
+ "BSN 123456782 en later weer 123456782",
286
+ { countries: ["NL"], referentialIntegrity: true }
287
+ );
288
+ console.log(result.redactedText);
289
+ // "BSN NATIONAL_ID_1 en later weer NATIONAL_ID_1"
290
+ ```
291
+
292
+ ## Architecture
293
+
294
+ ```
295
+ Input text
296
+ |
297
+ v
298
+ [Normalizer] -- Unicode normalization, whitespace cleanup
299
+ |
300
+ v
301
+ [Pass 1: Pattern Matching] -- All country + shared + custom regexes
302
+ |
303
+ v
304
+ [Pass 2a: Validation] -- Checksum validators (mod-97, Luhn, entropy, ...)
305
+ | Failed-validation spans become suppression zones
306
+ v
307
+ [Pass 2b: Suppression] -- Remove false positives (currency amounts, units,
308
+ | references, overlapping matches in suppression zones)
309
+ v
310
+ [Deduplication] -- Priority-aware: validated > custom > regex-only
311
+ | Within same tier, longer span wins
312
+ v
313
+ [Replacement] -- Right-to-left substitution with [ENTITY_TYPE] labels
314
+ | or labels
315
+ v
316
+ RedactResult
317
+ ```
318
+
319
+ ### Suppression Zones
320
+
321
+ When a regex matches a pattern that has a checksum validator but the checksum
322
+ fails, the matched span becomes a "suppression zone." Any purely regex-based
323
+ detection fully contained within that zone is suppressed as a false positive.
324
+
325
+ ### Deduplication Priority
326
+
327
+ 1. **Validated patterns** (checksum passes) -- highest priority
328
+ 2. **Custom patterns** (registered via `addCustomPattern()`)
329
+ 3. **Regex-only patterns** (no validator) -- lowest priority
330
+
331
+ Within the same tier, the longer span wins.
87
332
 
88
333
  ## CommonJS
89
334
 
@@ -91,6 +336,14 @@ AT, BE, BG, CH, CY, CZ, DE, DK, EE, EL, ES, FI, FR, HR, HU, IE, IS, IT, LT, LU,
91
336
  const { redact } = require("euredact");
92
337
  ```
93
338
 
339
+ ## Performance
340
+
341
+ | Metric | Value |
342
+ |---|---|
343
+ | Latency per redaction | ~0.02 ms |
344
+ | Package size | ~86 KB |
345
+ | Runtime dependencies | 0 |
346
+
94
347
  ## License
95
348
 
96
349
  Apache-2.0
package/dist/cjs/index.js CHANGED
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.COUNTRY_CONFIGS = exports.EuRedact = exports.DetectionSource = exports.EntityType = void 0;
4
4
  exports.availableCountries = availableCountries;
5
5
  exports.redact = redact;
6
+ exports.addCustomPattern = addCustomPattern;
6
7
  exports.redactBatch = redactBatch;
7
8
  var types_js_1 = require("./types.js");
8
9
  Object.defineProperty(exports, "EntityType", { enumerable: true, get: function () { return types_js_1.EntityType; } });
@@ -25,6 +26,9 @@ function availableCountries() {
25
26
  function redact(text, options) {
26
27
  return getInstance().redact(text, options);
27
28
  }
29
+ function addCustomPattern(name, pattern) {
30
+ getInstance().addCustomPattern(name, pattern);
31
+ }
28
32
  function redactBatch(texts, options) {
29
33
  return getInstance().redactBatch(texts, options);
30
34
  }
@@ -49,6 +49,56 @@ const SHARED = {
49
49
  p(types_js_1.EntityType.GPS_COORDINATES, String.raw `-?(?:[1-8]?\d(?:\.\d{4,})|90(?:\.0{4,}))\s*[,;/]\s*-?(?:1[0-7]\d(?:\.\d{4,})|180(?:\.0{4,})|\d{1,2}(?:\.\d{4,}))`),
50
50
  p(types_js_1.EntityType.UUID, String.raw `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b`),
51
51
  p(types_js_1.EntityType.SOCIAL_HANDLE, String.raw `(?<!\w)@[a-zA-Z][a-zA-Z0-9_.]{1,29}\b`),
52
+ // --- Secret / API Key (known prefixes — always active) ---
53
+ p(types_js_1.EntityType.SECRET, String.raw `\bAKIA[A-Z0-9]{16}\b`, null, "AWS Access Key ID"),
54
+ p(types_js_1.EntityType.SECRET, String.raw `\bgh[poas]_[a-zA-Z0-9]{36,}\b`, null, "GitHub token"),
55
+ p(types_js_1.EntityType.SECRET, String.raw `\bgithub_pat_[a-zA-Z0-9_]{22,}\b`, null, "GitHub fine-grained PAT"),
56
+ p(types_js_1.EntityType.SECRET, String.raw `\bgl[a-z]t-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab token"),
57
+ p(types_js_1.EntityType.SECRET, String.raw `\bglpat-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab PAT"),
58
+ p(types_js_1.EntityType.SECRET, String.raw `\b[sp]k_(?:live|test)_[a-zA-Z0-9]{24,}\b`, null, "Stripe key"),
59
+ p(types_js_1.EntityType.SECRET, String.raw `\bsk-(?:ant-)?[a-zA-Z0-9\-_]{20,}\b`, null, "OpenAI/Anthropic key"),
60
+ p(types_js_1.EntityType.SECRET, String.raw `\bxox[bpas]-\d[\d\-]{10,}`, null, "Slack numeric token"),
61
+ p(types_js_1.EntityType.SECRET, String.raw `\bxox[bpas]-[a-zA-Z0-9\-]{10,}`, null, "Slack token"),
62
+ p(types_js_1.EntityType.SECRET, String.raw `\beyJ[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\b`, null, "JWT token"),
63
+ p(types_js_1.EntityType.SECRET, String.raw `\bSG\.[a-zA-Z0-9_\-]{22}\.[a-zA-Z0-9_\-]{43}\b`, null, "SendGrid key"),
64
+ p(types_js_1.EntityType.SECRET, String.raw `\bhvs\.[a-zA-Z0-9_\-]{20,}\b`, null, "Vault token"),
65
+ p(types_js_1.EntityType.SECRET, String.raw `\bkey-[a-f0-9]{20,}\b`, null, "Mailgun key"),
66
+ p(types_js_1.EntityType.SECRET, String.raw `\bnpm_[a-zA-Z0-9]{36,}\b`, null, "npm token"),
67
+ p(types_js_1.EntityType.SECRET, String.raw `\bAIza[a-zA-Z0-9_\-]{35}\b`, null, "Google API key"),
68
+ p(types_js_1.EntityType.SECRET, String.raw `\bdop_v1_[a-f0-9]{64}\b`, null, "DigitalOcean token"),
69
+ p(types_js_1.EntityType.SECRET, String.raw `\bAC[a-f0-9]{32}\b`, null, "Twilio Account SID"),
70
+ p(types_js_1.EntityType.SECRET, String.raw `\bxapp-\d[\d\-]{20,}`, null, "Slack app numeric token"),
71
+ p(types_js_1.EntityType.SECRET, String.raw `\bxapp-[a-zA-Z0-9\-]{20,}`, null, "Slack app token"),
72
+ p(types_js_1.EntityType.SECRET, String.raw `\bdapi[a-f0-9]{32,}\b`, null, "Databricks PAT"),
73
+ // --- Secret / API Key (connection strings with embedded credentials) ---
74
+ p(types_js_1.EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]+:[^\s@]+@[^\s]+`, null, "Connection string with credentials"),
75
+ // --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
76
+ p(types_js_1.EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", [
77
+ "key", "token", "secret", "password", "credential",
78
+ "api_key", "apikey", "api-key", "auth", "bearer",
79
+ "aws_secret_access_key", "access_key", "private_key",
80
+ "wachtwoord", "mot de passe", "Passwort", "Schlüssel",
81
+ "clé", "sleutel", "lösenord", "hasło", "heslo",
82
+ "jelszó", "contraseña", "senha", "parola",
83
+ "AES_KEY", "ENCRYPTION_KEY", "SIGNING_KEY",
84
+ "ACCOUNT_SID", "AUTH_TOKEN", "TWILIO",
85
+ "RAILS_SECRET", "SECRET_KEY_BASE", "DATABASE_URL",
86
+ "NPM_TOKEN", "DIGITALOCEAN", "GOOGLE_API",
87
+ "kľúč", "nøkkel", "nyckel", "avain", "lykill",
88
+ "DB_PASS", "DB_PASSWORD", "DATASOURCE_PASSWORD",
89
+ "JWT_SIGNING", "SIGNING_KEY",
90
+ "passwd", "Kennwort",
91
+ ], true),
92
+ // --- Secret / API Key (entropy-based fallback for longer tokens) ---
93
+ p(types_js_1.EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", [
94
+ "key", "token", "secret", "password", "credential",
95
+ "api_key", "apikey", "api-key", "auth", "bearer",
96
+ "aws_secret_access_key", "access_key", "private_key",
97
+ "wachtwoord", "mot de passe", "Passwort", "Schlüssel",
98
+ "clé", "sleutel", "lösenord", "hasło", "heslo",
99
+ "jelszó", "contraseña", "senha", "parola",
100
+ "kľúč", "nøkkel", "nyckel", "avain", "lykill",
101
+ ], true),
52
102
  p(types_js_1.EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
53
103
  p(types_js_1.EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
54
104
  p(types_js_1.EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),
@@ -9,6 +9,14 @@ const index_js_1 = require("./countries/index.js");
9
9
  class RuleEngine {
10
10
  matcher = new matchers_js_1.MultiPatternMatcher();
11
11
  loadedCountries = new Set();
12
+ addCustomPattern(name, pattern) {
13
+ const pdef = {
14
+ entityType: name, pattern, validator: null,
15
+ description: "", contextKeywords: [], requiresContext: false,
16
+ };
17
+ this.matcher.addPattern(pdef, "CUSTOM");
18
+ this.matcher.compile();
19
+ }
12
20
  loadCountries(countryCodes) {
13
21
  let configs;
14
22
  if (countryCodes === null) {
@@ -21,7 +29,6 @@ class RuleEngine {
21
29
  const config = index_js_1.COUNTRY_CONFIGS[code];
22
30
  if (config)
23
31
  configs.push(config);
24
- // Silently skip unknown country codes (e.g. "INTL")
25
32
  }
26
33
  const shared = index_js_1.COUNTRY_CONFIGS["SHARED"];
27
34
  if (shared)
@@ -43,32 +50,66 @@ class RuleEngine {
43
50
  let rawMatches = this.matcher.scan(text);
44
51
  if (countryCodes !== null) {
45
52
  const codesUpper = new Set(countryCodes.map(c => c.toUpperCase()));
46
- rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED");
53
+ rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED" || m.countryCode === "CUSTOM");
54
+ }
55
+ // Matches that have a validator but fail it create suppression zones:
56
+ // the span is recognisably a specific entity (e.g. IBAN-shaped) so
57
+ // overlapping regex-only matches (license plate, phone) are false
58
+ // positives and must be suppressed.
59
+ const validated = [];
60
+ const suppressionZones = [];
61
+ for (const m of rawMatches) {
62
+ if (this.matcher.validate(m)) {
63
+ validated.push(m);
64
+ }
65
+ else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
66
+ suppressionZones.push([m.start, m.end]);
67
+ }
47
68
  }
48
- const validated = rawMatches.filter(m => this.matcher.validate(m));
49
- const detections = [];
69
+ // Build candidates with priority: validated (3) > custom (2) > regex-only (1)
70
+ const candidates = [];
50
71
  for (const match of validated) {
51
72
  if ((0, suppressors_js_1.shouldSuppress)(text, match))
52
73
  continue;
53
- detections.push({
54
- entityType: match.patternDef.entityType,
55
- start: match.start,
56
- end: match.end,
57
- text: match.text,
58
- source: types_js_1.DetectionSource.RULES,
59
- country: match.countryCode !== "SHARED" ? match.countryCode : null,
60
- confidence: "high",
74
+ if (match.patternDef.validator === null && suppressionZones.length > 0) {
75
+ if (suppressionZones.some(([zStart, zEnd]) => match.start >= zStart && match.end <= zEnd)) {
76
+ continue;
77
+ }
78
+ }
79
+ const hasValidator = match.patternDef.validator !== null;
80
+ const isValid = this.matcher.validate(match);
81
+ const priority = (hasValidator && isValid) ? 3 : match.countryCode === "CUSTOM" ? 2 : 1;
82
+ const spanLength = match.end - match.start;
83
+ const score = priority * 1_000_000 + spanLength;
84
+ candidates.push({
85
+ det: {
86
+ entityType: match.patternDef.entityType,
87
+ start: match.start,
88
+ end: match.end,
89
+ text: match.text,
90
+ source: types_js_1.DetectionSource.RULES,
91
+ country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM") ? match.countryCode : null,
92
+ confidence: "high",
93
+ },
94
+ score,
61
95
  });
62
96
  }
63
- detections.push(...(0, structural_js_1.detectStructuralDob)(text));
64
- return this.deduplicate(detections);
97
+ for (const d of (0, structural_js_1.detectStructuralDob)(text)) {
98
+ candidates.push({ det: d, score: 1_000_000 + (d.end - d.start) });
99
+ }
100
+ return this.deduplicate(candidates);
65
101
  }
66
- deduplicate(detections) {
67
- if (detections.length === 0)
68
- return detections;
69
- const sorted = [...detections].sort((a, b) => a.start - b.start || (b.end - b.start) - (a.end - a.start));
102
+ /**
103
+ * Remove overlapping detections with priority-aware resolution.
104
+ * Priority: validated (3) > custom (2) > regex-only (1).
105
+ * Within the same tier, longer span wins.
106
+ */
107
+ deduplicate(candidates) {
108
+ if (candidates.length === 0)
109
+ return [];
110
+ const sorted = [...candidates].sort((a, b) => b.score - a.score);
70
111
  const result = [];
71
- for (const det of sorted) {
112
+ for (const { det } of sorted) {
72
113
  let overlaps = false;
73
114
  for (const kept of result) {
74
115
  if (det.start < kept.end && det.end > kept.start) {
@@ -5,6 +5,11 @@ const validators_js_1 = require("./validators.js");
5
5
  class MultiPatternMatcher {
6
6
  patterns = [];
7
7
  compiled = false;
8
+ addPattern(patternDef, countryCode) {
9
+ const regex = new RegExp(patternDef.pattern, "gu");
10
+ this.patterns.push({ regex, patternDef, countryCode });
11
+ this.compiled = false;
12
+ }
8
13
  addCountry(config) {
9
14
  for (const pdef of config.patterns) {
10
15
  // Convert Python-style \b word boundaries — JS handles them natively
@@ -623,6 +623,35 @@ function validateGreekAmka(candidate) {
623
623
  }
624
624
  return total % 10 === 0;
625
625
  }
626
+ function validateHighEntropy(candidate) {
627
+ const clean = candidate.trim();
628
+ if (clean.length < 8)
629
+ return false;
630
+ // Short passwords (< 24 chars): require at least 1 letter, 1 digit, 1 special char
631
+ if (clean.length < 24) {
632
+ const hasLetter = /[a-zA-Z]/.test(clean);
633
+ const hasDigit = /\d/.test(clean);
634
+ const hasSpecial = /[^a-zA-Z0-9]/.test(clean);
635
+ if (hasLetter && hasDigit && hasSpecial)
636
+ return true;
637
+ // For short tokens without special chars, require entropy check
638
+ if (!hasSpecial && clean.length < 16)
639
+ return false;
640
+ }
641
+ const freq = new Map();
642
+ for (const ch of clean)
643
+ freq.set(ch, (freq.get(ch) ?? 0) + 1);
644
+ const len = clean.length;
645
+ let entropy = 0;
646
+ for (const c of freq.values()) {
647
+ const p = c / len;
648
+ entropy -= p * Math.log2(p);
649
+ }
650
+ // Hex-only strings have max ~4.0 bits entropy, use lower threshold
651
+ const isHex = /^[a-fA-F0-9]+$/.test(clean);
652
+ const threshold = isHex ? 3.0 : clean.length < 24 ? 3.0 : 3.5;
653
+ return entropy >= threshold;
654
+ }
626
655
  exports.VALIDATORS = {
627
656
  iban: validateIban,
628
657
  bsn: validateBsn,
@@ -666,4 +695,5 @@ exports.VALIDATORS = {
666
695
  uk_nhs: validateUkNhs,
667
696
  greek_afm: validateGreekAfm,
668
697
  greek_amka: validateGreekAmka,
698
+ high_entropy: validateHighEntropy,
669
699
  };
package/dist/cjs/sdk.js CHANGED
@@ -6,10 +6,10 @@ const normalizer_js_1 = require("./normalizer.js");
6
6
  const engine_js_1 = require("./rules/engine.js");
7
7
  const types_js_1 = require("./types.js");
8
8
  const DATE_TYPES = new Set([types_js_1.EntityType.DOB, types_js_1.EntityType.DATE_OF_DEATH]);
9
- class PseudonymMapper {
9
+ class ReferentialMapper {
10
10
  counters = new Map();
11
11
  mapping = new Map();
12
- getPseudonym(text, entityType) {
12
+ getLabel(text, entityType) {
13
13
  if (!this.mapping.has(text)) {
14
14
  const count = (this.counters.get(entityType) ?? 0) + 1;
15
15
  this.counters.set(entityType, count);
@@ -21,9 +21,13 @@ class PseudonymMapper {
21
21
  class EuRedact {
22
22
  engine = new engine_js_1.RuleEngine();
23
23
  cache = new cache_js_1.ResultCache();
24
- pseudonymMapper = new PseudonymMapper();
24
+ referentialMapper = new ReferentialMapper();
25
+ addCustomPattern(name, pattern) {
26
+ this.engine.addCustomPattern(name, pattern);
27
+ this.cache.clear();
28
+ }
25
29
  redact(text, options = {}) {
26
- const { countries = null, mode = "rules", pseudonymize = false, detectDates = false, cache = true, } = options;
30
+ const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
27
31
  const [normalizedText, offsetMapping] = (0, normalizer_js_1.normalize)(text);
28
32
  const countriesTuple = countries
29
33
  ? countries.map(c => c.toUpperCase()).sort()
@@ -51,8 +55,8 @@ class EuRedact {
51
55
  let redacted = text;
52
56
  for (let i = detections.length - 1; i >= 0; i--) {
53
57
  const det = detections[i];
54
- const replacement = pseudonymize
55
- ? this.pseudonymMapper.getPseudonym(det.text, det.entityType)
58
+ const replacement = referentialIntegrity
59
+ ? this.referentialMapper.getLabel(det.text, det.entityType)
56
60
  : `[${det.entityType}]`;
57
61
  redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
58
62
  }
package/dist/cjs/types.js CHANGED
@@ -32,6 +32,7 @@ var EntityType;
32
32
  EntityType["GPS_COORDINATES"] = "GPS_COORDINATES";
33
33
  EntityType["UUID"] = "UUID";
34
34
  EntityType["SOCIAL_HANDLE"] = "SOCIAL_HANDLE";
35
+ EntityType["SECRET"] = "SECRET";
35
36
  EntityType["OTHER"] = "OTHER";
36
37
  })(EntityType || (exports.EntityType = EntityType = {}));
37
38
  var DetectionSource;
@@ -6,4 +6,5 @@ import { type RedactOptions } from "./sdk.js";
6
6
  import type { RedactResult } from "./types.js";
7
7
  export declare function availableCountries(): string[];
8
8
  export declare function redact(text: string, options?: RedactOptions): RedactResult;
9
+ export declare function addCustomPattern(name: string, pattern: string): void;
9
10
  export declare function redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
package/dist/esm/index.js CHANGED
@@ -15,6 +15,9 @@ export function availableCountries() {
15
15
  export function redact(text, options) {
16
16
  return getInstance().redact(text, options);
17
17
  }
18
+ export function addCustomPattern(name, pattern) {
19
+ getInstance().addCustomPattern(name, pattern);
20
+ }
18
21
  export function redactBatch(texts, options) {
19
22
  return getInstance().redactBatch(texts, options);
20
23
  }
@@ -46,6 +46,56 @@ const SHARED = {
46
46
  p(EntityType.GPS_COORDINATES, String.raw `-?(?:[1-8]?\d(?:\.\d{4,})|90(?:\.0{4,}))\s*[,;/]\s*-?(?:1[0-7]\d(?:\.\d{4,})|180(?:\.0{4,})|\d{1,2}(?:\.\d{4,}))`),
47
47
  p(EntityType.UUID, String.raw `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b`),
48
48
  p(EntityType.SOCIAL_HANDLE, String.raw `(?<!\w)@[a-zA-Z][a-zA-Z0-9_.]{1,29}\b`),
49
+ // --- Secret / API Key (known prefixes — always active) ---
50
+ p(EntityType.SECRET, String.raw `\bAKIA[A-Z0-9]{16}\b`, null, "AWS Access Key ID"),
51
+ p(EntityType.SECRET, String.raw `\bgh[poas]_[a-zA-Z0-9]{36,}\b`, null, "GitHub token"),
52
+ p(EntityType.SECRET, String.raw `\bgithub_pat_[a-zA-Z0-9_]{22,}\b`, null, "GitHub fine-grained PAT"),
53
+ p(EntityType.SECRET, String.raw `\bgl[a-z]t-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab token"),
54
+ p(EntityType.SECRET, String.raw `\bglpat-[a-zA-Z0-9_\-]{20,}\b`, null, "GitLab PAT"),
55
+ p(EntityType.SECRET, String.raw `\b[sp]k_(?:live|test)_[a-zA-Z0-9]{24,}\b`, null, "Stripe key"),
56
+ p(EntityType.SECRET, String.raw `\bsk-(?:ant-)?[a-zA-Z0-9\-_]{20,}\b`, null, "OpenAI/Anthropic key"),
57
+ p(EntityType.SECRET, String.raw `\bxox[bpas]-\d[\d\-]{10,}`, null, "Slack numeric token"),
58
+ p(EntityType.SECRET, String.raw `\bxox[bpas]-[a-zA-Z0-9\-]{10,}`, null, "Slack token"),
59
+ p(EntityType.SECRET, String.raw `\beyJ[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\b`, null, "JWT token"),
60
+ p(EntityType.SECRET, String.raw `\bSG\.[a-zA-Z0-9_\-]{22}\.[a-zA-Z0-9_\-]{43}\b`, null, "SendGrid key"),
61
+ p(EntityType.SECRET, String.raw `\bhvs\.[a-zA-Z0-9_\-]{20,}\b`, null, "Vault token"),
62
+ p(EntityType.SECRET, String.raw `\bkey-[a-f0-9]{20,}\b`, null, "Mailgun key"),
63
+ p(EntityType.SECRET, String.raw `\bnpm_[a-zA-Z0-9]{36,}\b`, null, "npm token"),
64
+ p(EntityType.SECRET, String.raw `\bAIza[a-zA-Z0-9_\-]{35}\b`, null, "Google API key"),
65
+ p(EntityType.SECRET, String.raw `\bdop_v1_[a-f0-9]{64}\b`, null, "DigitalOcean token"),
66
+ p(EntityType.SECRET, String.raw `\bAC[a-f0-9]{32}\b`, null, "Twilio Account SID"),
67
+ p(EntityType.SECRET, String.raw `\bxapp-\d[\d\-]{20,}`, null, "Slack app numeric token"),
68
+ p(EntityType.SECRET, String.raw `\bxapp-[a-zA-Z0-9\-]{20,}`, null, "Slack app token"),
69
+ p(EntityType.SECRET, String.raw `\bdapi[a-f0-9]{32,}\b`, null, "Databricks PAT"),
70
+ // --- Secret / API Key (connection strings with embedded credentials) ---
71
+ p(EntityType.SECRET, String.raw `(?:mongodb|mysql|postgres(?:ql)?|redis|amqp|rabbitmq):\/\/[^\s:]+:[^\s@]+@[^\s]+`, null, "Connection string with credentials"),
72
+ // --- Secret / API Key (assignment-based: KEY=value or KEY: value) ---
73
+ p(EntityType.SECRET, String.raw `(?<=[:=] )\S{8,}|(?<=[:=])\S{8,}`, "high_entropy", "Assigned secret value", [
74
+ "key", "token", "secret", "password", "credential",
75
+ "api_key", "apikey", "api-key", "auth", "bearer",
76
+ "aws_secret_access_key", "access_key", "private_key",
77
+ "wachtwoord", "mot de passe", "Passwort", "Schlüssel",
78
+ "clé", "sleutel", "lösenord", "hasło", "heslo",
79
+ "jelszó", "contraseña", "senha", "parola",
80
+ "AES_KEY", "ENCRYPTION_KEY", "SIGNING_KEY",
81
+ "ACCOUNT_SID", "AUTH_TOKEN", "TWILIO",
82
+ "RAILS_SECRET", "SECRET_KEY_BASE", "DATABASE_URL",
83
+ "NPM_TOKEN", "DIGITALOCEAN", "GOOGLE_API",
84
+ "kľúč", "nøkkel", "nyckel", "avain", "lykill",
85
+ "DB_PASS", "DB_PASSWORD", "DATASOURCE_PASSWORD",
86
+ "JWT_SIGNING", "SIGNING_KEY",
87
+ "passwd", "Kennwort",
88
+ ], true),
89
+ // --- Secret / API Key (entropy-based fallback for longer tokens) ---
90
+ p(EntityType.SECRET, String.raw `\b[A-Za-z0-9_\-+/]{24,}[A-Za-z0-9_\-+/=]*\b`, "high_entropy", "High-entropy token", [
91
+ "key", "token", "secret", "password", "credential",
92
+ "api_key", "apikey", "api-key", "auth", "bearer",
93
+ "aws_secret_access_key", "access_key", "private_key",
94
+ "wachtwoord", "mot de passe", "Passwort", "Schlüssel",
95
+ "clé", "sleutel", "lösenord", "hasło", "heslo",
96
+ "jelszó", "contraseña", "senha", "parola",
97
+ "kľúč", "nøkkel", "nyckel", "avain", "lykill",
98
+ ], true),
49
99
  p(EntityType.DOB, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "DD/MM/YYYY", DOB_CONTEXT, true),
50
100
  p(EntityType.DATE_OF_DEATH, String.raw `\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b`, null, "", DATE_OF_DEATH_CONTEXT, true),
51
101
  p(EntityType.DOB, String.raw `\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b`, null, "ISO YYYY-MM-DD", DOB_ISO_CONTEXT, true),
@@ -2,8 +2,14 @@ import { type Detection } from "../types.js";
2
2
  export declare class RuleEngine {
3
3
  private matcher;
4
4
  private loadedCountries;
5
+ addCustomPattern(name: string, pattern: string): void;
5
6
  loadCountries(countryCodes: string[] | null): void;
6
7
  detect(text: string, countryCodes: string[] | null): Detection[];
8
+ /**
9
+ * Remove overlapping detections with priority-aware resolution.
10
+ * Priority: validated (3) > custom (2) > regex-only (1).
11
+ * Within the same tier, longer span wins.
12
+ */
7
13
  private deduplicate;
8
14
  getLoadedCountries(): Set<string>;
9
15
  }
@@ -6,6 +6,14 @@ import { COUNTRY_CONFIGS } from "./countries/index.js";
6
6
  export class RuleEngine {
7
7
  matcher = new MultiPatternMatcher();
8
8
  loadedCountries = new Set();
9
+ addCustomPattern(name, pattern) {
10
+ const pdef = {
11
+ entityType: name, pattern, validator: null,
12
+ description: "", contextKeywords: [], requiresContext: false,
13
+ };
14
+ this.matcher.addPattern(pdef, "CUSTOM");
15
+ this.matcher.compile();
16
+ }
9
17
  loadCountries(countryCodes) {
10
18
  let configs;
11
19
  if (countryCodes === null) {
@@ -18,7 +26,6 @@ export class RuleEngine {
18
26
  const config = COUNTRY_CONFIGS[code];
19
27
  if (config)
20
28
  configs.push(config);
21
- // Silently skip unknown country codes (e.g. "INTL")
22
29
  }
23
30
  const shared = COUNTRY_CONFIGS["SHARED"];
24
31
  if (shared)
@@ -40,32 +47,66 @@ export class RuleEngine {
40
47
  let rawMatches = this.matcher.scan(text);
41
48
  if (countryCodes !== null) {
42
49
  const codesUpper = new Set(countryCodes.map(c => c.toUpperCase()));
43
- rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED");
50
+ rawMatches = rawMatches.filter(m => codesUpper.has(m.countryCode) || m.countryCode === "SHARED" || m.countryCode === "CUSTOM");
51
+ }
52
+ // Matches that have a validator but fail it create suppression zones:
53
+ // the span is recognisably a specific entity (e.g. IBAN-shaped) so
54
+ // overlapping regex-only matches (license plate, phone) are false
55
+ // positives and must be suppressed.
56
+ const validated = [];
57
+ const suppressionZones = [];
58
+ for (const m of rawMatches) {
59
+ if (this.matcher.validate(m)) {
60
+ validated.push(m);
61
+ }
62
+ else if (m.patternDef.validator !== null && !m.patternDef.requiresContext) {
63
+ suppressionZones.push([m.start, m.end]);
64
+ }
44
65
  }
45
- const validated = rawMatches.filter(m => this.matcher.validate(m));
46
- const detections = [];
66
+ // Build candidates with priority: validated (3) > custom (2) > regex-only (1)
67
+ const candidates = [];
47
68
  for (const match of validated) {
48
69
  if (shouldSuppress(text, match))
49
70
  continue;
50
- detections.push({
51
- entityType: match.patternDef.entityType,
52
- start: match.start,
53
- end: match.end,
54
- text: match.text,
55
- source: DetectionSource.RULES,
56
- country: match.countryCode !== "SHARED" ? match.countryCode : null,
57
- confidence: "high",
71
+ if (match.patternDef.validator === null && suppressionZones.length > 0) {
72
+ if (suppressionZones.some(([zStart, zEnd]) => match.start >= zStart && match.end <= zEnd)) {
73
+ continue;
74
+ }
75
+ }
76
+ const hasValidator = match.patternDef.validator !== null;
77
+ const isValid = this.matcher.validate(match);
78
+ const priority = (hasValidator && isValid) ? 3 : match.countryCode === "CUSTOM" ? 2 : 1;
79
+ const spanLength = match.end - match.start;
80
+ const score = priority * 1_000_000 + spanLength;
81
+ candidates.push({
82
+ det: {
83
+ entityType: match.patternDef.entityType,
84
+ start: match.start,
85
+ end: match.end,
86
+ text: match.text,
87
+ source: DetectionSource.RULES,
88
+ country: (match.countryCode !== "SHARED" && match.countryCode !== "CUSTOM") ? match.countryCode : null,
89
+ confidence: "high",
90
+ },
91
+ score,
58
92
  });
59
93
  }
60
- detections.push(...detectStructuralDob(text));
61
- return this.deduplicate(detections);
94
+ for (const d of detectStructuralDob(text)) {
95
+ candidates.push({ det: d, score: 1_000_000 + (d.end - d.start) });
96
+ }
97
+ return this.deduplicate(candidates);
62
98
  }
63
- deduplicate(detections) {
64
- if (detections.length === 0)
65
- return detections;
66
- const sorted = [...detections].sort((a, b) => a.start - b.start || (b.end - b.start) - (a.end - a.start));
99
+ /**
100
+ * Remove overlapping detections with priority-aware resolution.
101
+ * Priority: validated (3) > custom (2) > regex-only (1).
102
+ * Within the same tier, longer span wins.
103
+ */
104
+ deduplicate(candidates) {
105
+ if (candidates.length === 0)
106
+ return [];
107
+ const sorted = [...candidates].sort((a, b) => b.score - a.score);
67
108
  const result = [];
68
- for (const det of sorted) {
109
+ for (const { det } of sorted) {
69
110
  let overlaps = false;
70
111
  for (const kept of result) {
71
112
  if (det.start < kept.end && det.end > kept.start) {
@@ -1,9 +1,10 @@
1
- import type { CountryConfig } from "../types.js";
1
+ import type { CountryConfig, PatternDef } from "../types.js";
2
2
  import type { RawMatch } from "./suppressors.js";
3
3
  export type { RawMatch };
4
4
  export declare class MultiPatternMatcher {
5
5
  private patterns;
6
6
  private compiled;
7
+ addPattern(patternDef: PatternDef, countryCode: string): void;
7
8
  addCountry(config: CountryConfig): void;
8
9
  compile(): void;
9
10
  scan(text: string): RawMatch[];
@@ -2,6 +2,11 @@ import { VALIDATORS } from "./validators.js";
2
2
  export class MultiPatternMatcher {
3
3
  patterns = [];
4
4
  compiled = false;
5
+ addPattern(patternDef, countryCode) {
6
+ const regex = new RegExp(patternDef.pattern, "gu");
7
+ this.patterns.push({ regex, patternDef, countryCode });
8
+ this.compiled = false;
9
+ }
5
10
  addCountry(config) {
6
11
  for (const pdef of config.patterns) {
7
12
  // Convert Python-style \b word boundaries — JS handles them natively
@@ -578,6 +578,35 @@ export function validateGreekAmka(candidate) {
578
578
  }
579
579
  return total % 10 === 0;
580
580
  }
581
+ function validateHighEntropy(candidate) {
582
+ const clean = candidate.trim();
583
+ if (clean.length < 8)
584
+ return false;
585
+ // Short passwords (< 24 chars): require at least 1 letter, 1 digit, 1 special char
586
+ if (clean.length < 24) {
587
+ const hasLetter = /[a-zA-Z]/.test(clean);
588
+ const hasDigit = /\d/.test(clean);
589
+ const hasSpecial = /[^a-zA-Z0-9]/.test(clean);
590
+ if (hasLetter && hasDigit && hasSpecial)
591
+ return true;
592
+ // For short tokens without special chars, require entropy check
593
+ if (!hasSpecial && clean.length < 16)
594
+ return false;
595
+ }
596
+ const freq = new Map();
597
+ for (const ch of clean)
598
+ freq.set(ch, (freq.get(ch) ?? 0) + 1);
599
+ const len = clean.length;
600
+ let entropy = 0;
601
+ for (const c of freq.values()) {
602
+ const p = c / len;
603
+ entropy -= p * Math.log2(p);
604
+ }
605
+ // Hex-only strings have max ~4.0 bits entropy, use lower threshold
606
+ const isHex = /^[a-fA-F0-9]+$/.test(clean);
607
+ const threshold = isHex ? 3.0 : clean.length < 24 ? 3.0 : 3.5;
608
+ return entropy >= threshold;
609
+ }
581
610
  export const VALIDATORS = {
582
611
  iban: validateIban,
583
612
  bsn: validateBsn,
@@ -621,4 +650,5 @@ export const VALIDATORS = {
621
650
  uk_nhs: validateUkNhs,
622
651
  greek_afm: validateGreekAfm,
623
652
  greek_amka: validateGreekAmka,
653
+ high_entropy: validateHighEntropy,
624
654
  };
package/dist/esm/sdk.d.ts CHANGED
@@ -2,14 +2,15 @@ import { type RedactResult } from "./types.js";
2
2
  export interface RedactOptions {
3
3
  countries?: string[] | null;
4
4
  mode?: string;
5
- pseudonymize?: boolean;
5
+ referentialIntegrity?: boolean;
6
6
  detectDates?: boolean;
7
7
  cache?: boolean;
8
8
  }
9
9
  export declare class EuRedact {
10
10
  private engine;
11
11
  private cache;
12
- private pseudonymMapper;
12
+ private referentialMapper;
13
+ addCustomPattern(name: string, pattern: string): void;
13
14
  redact(text: string, options?: RedactOptions): RedactResult;
14
15
  redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
15
16
  }
package/dist/esm/sdk.js CHANGED
@@ -3,10 +3,10 @@ import { normalize, mapOffsetToOriginal } from "./normalizer.js";
3
3
  import { RuleEngine } from "./rules/engine.js";
4
4
  import { EntityType } from "./types.js";
5
5
  const DATE_TYPES = new Set([EntityType.DOB, EntityType.DATE_OF_DEATH]);
6
- class PseudonymMapper {
6
+ class ReferentialMapper {
7
7
  counters = new Map();
8
8
  mapping = new Map();
9
- getPseudonym(text, entityType) {
9
+ getLabel(text, entityType) {
10
10
  if (!this.mapping.has(text)) {
11
11
  const count = (this.counters.get(entityType) ?? 0) + 1;
12
12
  this.counters.set(entityType, count);
@@ -18,9 +18,13 @@ class PseudonymMapper {
18
18
  export class EuRedact {
19
19
  engine = new RuleEngine();
20
20
  cache = new ResultCache();
21
- pseudonymMapper = new PseudonymMapper();
21
+ referentialMapper = new ReferentialMapper();
22
+ addCustomPattern(name, pattern) {
23
+ this.engine.addCustomPattern(name, pattern);
24
+ this.cache.clear();
25
+ }
22
26
  redact(text, options = {}) {
23
- const { countries = null, mode = "rules", pseudonymize = false, detectDates = false, cache = true, } = options;
27
+ const { countries = null, mode = "rules", referentialIntegrity = false, detectDates = false, cache = true, } = options;
24
28
  const [normalizedText, offsetMapping] = normalize(text);
25
29
  const countriesTuple = countries
26
30
  ? countries.map(c => c.toUpperCase()).sort()
@@ -48,8 +52,8 @@ export class EuRedact {
48
52
  let redacted = text;
49
53
  for (let i = detections.length - 1; i >= 0; i--) {
50
54
  const det = detections[i];
51
- const replacement = pseudonymize
52
- ? this.pseudonymMapper.getPseudonym(det.text, det.entityType)
55
+ const replacement = referentialIntegrity
56
+ ? this.referentialMapper.getLabel(det.text, det.entityType)
53
57
  : `[${det.entityType}]`;
54
58
  redacted = redacted.slice(0, det.start) + replacement + redacted.slice(det.end);
55
59
  }
@@ -28,6 +28,7 @@ export declare enum EntityType {
28
28
  GPS_COORDINATES = "GPS_COORDINATES",
29
29
  UUID = "UUID",
30
30
  SOCIAL_HANDLE = "SOCIAL_HANDLE",
31
+ SECRET = "SECRET",
31
32
  OTHER = "OTHER"
32
33
  }
33
34
  export declare enum DetectionSource {
@@ -35,7 +36,7 @@ export declare enum DetectionSource {
35
36
  CLOUD = "cloud"
36
37
  }
37
38
  export interface Detection {
38
- entityType: EntityType;
39
+ entityType: EntityType | string;
39
40
  start: number;
40
41
  end: number;
41
42
  text: string;
@@ -50,7 +51,7 @@ export interface RedactResult {
50
51
  degraded: boolean;
51
52
  }
52
53
  export interface PatternDef {
53
- entityType: EntityType;
54
+ entityType: EntityType | string;
54
55
  pattern: string;
55
56
  validator: string | null;
56
57
  description: string;
package/dist/esm/types.js CHANGED
@@ -29,6 +29,7 @@ export var EntityType;
29
29
  EntityType["GPS_COORDINATES"] = "GPS_COORDINATES";
30
30
  EntityType["UUID"] = "UUID";
31
31
  EntityType["SOCIAL_HANDLE"] = "SOCIAL_HANDLE";
32
+ EntityType["SECRET"] = "SECRET";
32
33
  EntityType["OTHER"] = "OTHER";
33
34
  })(EntityType || (EntityType = {}));
34
35
  export var DetectionSource;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "euredact",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -46,7 +46,7 @@
46
46
  ],
47
47
  "repository": {
48
48
  "type": "git",
49
- "url": "https://github.com/jnjs/euredact"
49
+ "url": "https://github.com/euredact/euredact"
50
50
  },
51
51
  "scripts": {
52
52
  "build:esm": "tsc",