euredact 0.3.9 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -9
- package/dist/cjs/cloud/client.js +219 -0
- package/dist/cjs/cloud/config.js +57 -0
- package/dist/cjs/cloud/errors.js +55 -0
- package/dist/cjs/cloud/index.js +31 -0
- package/dist/cjs/index.js +22 -3
- package/dist/cjs/sdk.js +92 -0
- package/dist/cjs/types.js +45 -1
- package/dist/esm/cloud/client.d.ts +42 -0
- package/dist/esm/cloud/client.js +215 -0
- package/dist/esm/cloud/config.d.ts +43 -0
- package/dist/esm/cloud/config.js +51 -0
- package/dist/esm/cloud/errors.d.ts +32 -0
- package/dist/esm/cloud/errors.js +48 -0
- package/dist/esm/cloud/index.d.ts +19 -0
- package/dist/esm/cloud/index.js +19 -0
- package/dist/esm/index.d.ts +9 -1
- package/dist/esm/index.js +11 -1
- package/dist/esm/sdk.d.ts +7 -0
- package/dist/esm/sdk.js +59 -0
- package/dist/esm/types.d.ts +33 -1
- package/dist/esm/types.js +44 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -34,7 +34,8 @@ console.log(result.detections);
|
|
|
34
34
|
## Features
|
|
35
35
|
|
|
36
36
|
- **31 European countries** with country-specific patterns
|
|
37
|
-
- **
|
|
37
|
+
- **41 PII entity types** (30 from the rules engine, 11 from the
|
|
38
|
+
[cloud tier](#cloud-tier)): national IDs, IBANs, phone numbers, email, VAT
|
|
38
39
|
numbers, license plates, credit cards, BIC/SWIFT, VIN, IMEI, GPS coordinates,
|
|
39
40
|
UUIDs, social handles, IP/IPv6, MAC addresses, secrets/API keys, and more
|
|
40
41
|
- **Secret/API key detection:** known-prefix patterns for AWS, GitHub, Stripe,
|
|
@@ -218,7 +219,7 @@ interface RedactResult {
|
|
|
218
219
|
String enum with all supported PII categories:
|
|
219
220
|
|
|
220
221
|
```
|
|
221
|
-
|
|
222
|
+
PERSON_NAME ADDRESS BANK_ACCOUNT BIC
|
|
222
223
|
CREDIT_CARD PHONE EMAIL DOB
|
|
223
224
|
DATE_OF_DEATH NATIONAL_ID SSN TAX_ID
|
|
224
225
|
PASSPORT DRIVERS_LICENSE RESIDENCE_PERMIT LICENSE_PLATE
|
|
@@ -226,6 +227,13 @@ VIN VAT POSTAL_CODE IP_ADDRESS
|
|
|
226
227
|
IPV6_ADDRESS MAC_ADDRESS HEALTH_INSURANCE HEALTHCARE_PROVIDER
|
|
227
228
|
CHAMBER_OF_COMMERCE IMEI GPS_COORDINATES UUID
|
|
228
229
|
SOCIAL_HANDLE SECRET INTERNAL_ID OTHER
|
|
230
|
+
|
|
231
|
+
Cloud tier only — the rule engine never emits these, because there is no shape
|
|
232
|
+
to match on. That is precisely why the model exists:
|
|
233
|
+
|
|
234
|
+
ORGANISATION_NAME JOB_TITLE MEDICAL_CONDITION SENSITIVE_ATTRIBUTE
|
|
235
|
+
BIOMETRIC_REF FINANCIAL_AMOUNT QUASI_IDENTIFIER CREDENTIAL
|
|
236
|
+
URL
|
|
229
237
|
```
|
|
230
238
|
|
|
231
239
|
`INTERNAL_ID` — an employee, badge or customer number tied to a person — is
|
|
@@ -264,6 +272,65 @@ masked is unchanged either way.
|
|
|
264
272
|
|
|
265
273
|
For custom patterns, `entityType` is a plain string (e.g. `"EMPLOYEE_ID"`).
|
|
266
274
|
|
|
275
|
+
## Cloud tier
|
|
276
|
+
|
|
277
|
+
> **Status: private alpha.** The cloud tier is in closed testing — it is **not**
|
|
278
|
+
> in public beta and is not generally available. Keys are issued to alpha
|
|
279
|
+
> participants only, and the request/response surface may still change between
|
|
280
|
+
> releases. The rules engine below is unaffected and is the supported path:
|
|
281
|
+
> `mode="rules"` is the default and carries no alpha caveat.
|
|
282
|
+
|
|
283
|
+
The rule engine catches what has a shape: IBANs, national IDs, phone numbers,
|
|
284
|
+
anything with a checksum. It cannot catch what does not — a person's name, an
|
|
285
|
+
employer, a diagnosis, a job title. The cloud tier adds a fine-tuned model
|
|
286
|
+
asked only *what did the rules miss?*
|
|
287
|
+
|
|
288
|
+
```ts
|
|
289
|
+
import { configure, redactAsync } from "euredact";
|
|
290
|
+
|
|
291
|
+
configure({ apiKey: "erk_..." }); // or set EUREDACT_API_KEY
|
|
292
|
+
const result = await redactAsync(text, { countries: ["BE"], mode: "cloud" });
|
|
293
|
+
|
|
294
|
+
result.source; // "cloud"
|
|
295
|
+
result.detections.map(d => [d.entityType, d.text]);
|
|
296
|
+
// [["PERSON_NAME", "Bas Verhoeven"], ["PHONE", "+32 ..."]]
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
**`redact()` is synchronous and cannot serve the cloud tier**, so
|
|
300
|
+
`redact(text, { mode: "cloud" })` throws and names `redactAsync`. It never
|
|
301
|
+
falls back to rules-only output: a caller who believes names and diagnoses were
|
|
302
|
+
checked, and ships a document that only had its phone numbers masked, is the
|
|
303
|
+
one failure this library must not have.
|
|
304
|
+
|
|
305
|
+
`redactAsync` with `mode: "rules"` resolves immediately with exactly what
|
|
306
|
+
`redact()` returns, so a caller that may or may not use the cloud tier can hold
|
|
307
|
+
one code path.
|
|
308
|
+
|
|
309
|
+
Retries carry an `Idempotency-Key`, so a retry after a timeout cannot bill
|
|
310
|
+
twice. `Retry-After` is obeyed. A document that outlives the service's sync
|
|
311
|
+
window is polled transparently. Oversized input rejects with `TooLargeError`
|
|
312
|
+
(413): the service refuses it rather than chunking, because the model has never
|
|
313
|
+
seen a chunk boundary.
|
|
314
|
+
|
|
315
|
+
Options the service cannot honour reject rather than being ignored: multiple
|
|
316
|
+
`countries`, `countryHint`, `context`/`chunkOffset` and `referentialIntegrity`.
|
|
317
|
+
|
|
318
|
+
The package stays **zero-dependency** — the client uses the platform's own
|
|
319
|
+
`fetch`. Node 18+ provides one; on Node 16 the rules engine is unaffected and a
|
|
320
|
+
`fetchImpl` can be supplied.
|
|
321
|
+
|
|
322
|
+
## `NAME` is now `PERSON_NAME`
|
|
323
|
+
|
|
324
|
+
The canonical type name is `PERSON_NAME`; `NAME` is a legacy alias, exactly as
|
|
325
|
+
`IBAN` aliases `BANK_ACCOUNT`. The placeholder written into redacted text is
|
|
326
|
+
`[PERSON_NAME]`.
|
|
327
|
+
|
|
328
|
+
Nothing could have depended on the old value: the type is cloud-only and the
|
|
329
|
+
cloud tier was stubbed until this release, so it was never emitted. Code
|
|
330
|
+
matching the *string* `"NAME"` should be updated; `LEGACY_TYPE_ALIASES`
|
|
331
|
+
publishes the mapping, and `STREET_ADDRESS` → `ADDRESS` and
|
|
332
|
+
`NATIONALITY_ETHNICITY` → `SENSITIVE_ATTRIBUTE` are recognised the same way.
|
|
333
|
+
|
|
267
334
|
## Country codes
|
|
268
335
|
|
|
269
336
|
`countries` accepts **ISO 3166-1 alpha-2** codes. The two EU/VAT spellings are
|
|
@@ -314,8 +381,9 @@ redact("Rekening: BE68 5390 0754 7034", { countries: ["AT"] });
|
|
|
314
381
|
### Which country a value belongs to
|
|
315
382
|
|
|
316
383
|
Because every pattern runs, the same digits often match several countries'
|
|
317
|
-
schemes.
|
|
318
|
-
country's
|
|
384
|
+
schemes. Of the national-ID values in our corpus that pass any country's checksum,
|
|
385
|
+
34.7% pass more than one country's (32,827 of 94,528), so the digits alone
|
|
386
|
+
cannot decide it — the *document* does.
|
|
319
387
|
|
|
320
388
|
The engine infers the document's countries from entities that carry their
|
|
321
389
|
country in the string, then uses that to resolve the ambiguity:
|
|
@@ -623,18 +691,24 @@ Two tiers are less obvious than they look:
|
|
|
623
691
|
|
|
624
692
|
## Performance
|
|
625
693
|
|
|
626
|
-
Measured on one core
|
|
627
|
-
|
|
694
|
+
Measured on one core (Apple Silicon M3 Pro, Node 22.12), all 31 countries
|
|
695
|
+
loaded, `detectDates: true`, cache off, over two cohorts sampled evenly from the
|
|
696
|
+
corpus: 3,000 distinct short records and 611 distinct real documents. Median per
|
|
697
|
+
document, 10th–90th percentile in brackets:
|
|
628
698
|
|
|
629
699
|
| Input | Latency | Throughput |
|
|
630
700
|
|---|---:|---:|
|
|
631
|
-
| Short record (
|
|
632
|
-
| Real document (3,
|
|
701
|
+
| Short record (~190 chars) | 154 µs <br><sub>89 – 237</sub> | 6,494 docs/s |
|
|
702
|
+
| Real document (~3,450 chars) | 1.43 ms <br><sub>1.04 – 1.96</sub> | 699 docs/s |
|
|
703
|
+
|
|
704
|
+
Cost tracks identifier density rather than length: the per-candidate cue and
|
|
705
|
+
suppressor checks dominate, so a form-like document dense in national IDs costs
|
|
706
|
+
several times an ordinary chat log of the same size.
|
|
633
707
|
|
|
634
708
|
No optional accelerator is needed or offered. The Python SDK ships an
|
|
635
709
|
`[fast]` extra (RE2 / Aho-Corasick) because CPython's regex engine is the
|
|
636
710
|
bottleneck there; V8's has literal prefilters that make it unnecessary here.
|
|
637
|
-
Measured on the same 611 documents, this SDK runs about
|
|
711
|
+
Measured on the same 611 documents, this SDK runs about 6–7× faster than the
|
|
638
712
|
accelerated Python path, so adding a native addon — and with it the loss of
|
|
639
713
|
bundler, edge-runtime and Deno compatibility — would buy nothing.
|
|
640
714
|
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* [CLOUD EXTENSION] The cloud client.
|
|
4
|
+
*
|
|
5
|
+
* Speaks to the euRedact inference service: a deterministic rules engine
|
|
6
|
+
* followed by a fine-tuned model asked only *what did the rules miss?* The
|
|
7
|
+
* service returns both the redacted document and located spans, so this
|
|
8
|
+
* module's job is transport and translation, not detection.
|
|
9
|
+
*
|
|
10
|
+
* Three things it hides from the caller:
|
|
11
|
+
*
|
|
12
|
+
* - **Whether the document finished inside the service's sync window.** A short
|
|
13
|
+
* document comes back on the original request; a long one, or one arriving
|
|
14
|
+
* while a GPU node is cold, returns a job handle and is polled. Callers
|
|
15
|
+
* should never write that branch, so `redact()` resolves either way.
|
|
16
|
+
* - **Retries.** Every request carries an `Idempotency-Key`, so a retry after a
|
|
17
|
+
* timeout cannot produce a second job or a second usage row.
|
|
18
|
+
* - **`Retry-After`.** When the service says how long to wait, that is obeyed
|
|
19
|
+
* rather than second-guessed with a backoff curve.
|
|
20
|
+
*
|
|
21
|
+
* Zero dependencies, like the rest of the package: this uses the platform's
|
|
22
|
+
* own `fetch`.
|
|
23
|
+
*/
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.CloudClient = void 0;
|
|
26
|
+
const config_js_1 = require("./config.js");
|
|
27
|
+
const errors_js_1 = require("./errors.js");
|
|
28
|
+
const types_js_1 = require("../types.js");
|
|
29
|
+
const RETRY_STATUS = new Set([429, 500, 502, 503, 504]);
|
|
30
|
+
const MAX_BACKOFF_MS = 30_000;
|
|
31
|
+
function requireFetch(supplied) {
|
|
32
|
+
const impl = supplied ?? globalThis.fetch;
|
|
33
|
+
if (typeof impl !== "function") {
|
|
34
|
+
throw new errors_js_1.NotConfiguredError("the cloud tier needs a global fetch. Node 18+ provides one; on Node 16 " +
|
|
35
|
+
"supply one via the fetchImpl option or install a polyfill.");
|
|
36
|
+
}
|
|
37
|
+
return impl;
|
|
38
|
+
}
|
|
39
|
+
function sleep(ms) {
|
|
40
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
41
|
+
}
|
|
42
|
+
function retryAfterMs(headers) {
|
|
43
|
+
const raw = headers?.get("Retry-After");
|
|
44
|
+
if (!raw)
|
|
45
|
+
return null;
|
|
46
|
+
const seconds = Number(raw);
|
|
47
|
+
// The HTTP-date form is legal too; fall back to backoff rather than parse it.
|
|
48
|
+
return Number.isFinite(seconds) ? Math.max(0, seconds * 1000) : null;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Full jitter. A synchronised retry storm from many clients is how a
|
|
52
|
+
* recovering service is knocked back over.
|
|
53
|
+
*/
|
|
54
|
+
function backoffMs(attempt, retryAfter) {
|
|
55
|
+
if (retryAfter !== null)
|
|
56
|
+
return Math.min(retryAfter, MAX_BACKOFF_MS);
|
|
57
|
+
return Math.min(MAX_BACKOFF_MS, Math.random() * 2 ** attempt * 1000);
|
|
58
|
+
}
|
|
59
|
+
function uuid() {
|
|
60
|
+
const c = globalThis.crypto;
|
|
61
|
+
if (c && typeof c.randomUUID === "function")
|
|
62
|
+
return c.randomUUID();
|
|
63
|
+
return `idem-${Date.now().toString(16)}-${Math.random().toString(16).slice(2)}`;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Canon name -> EntityType, tolerating one this build has not heard of.
|
|
67
|
+
*
|
|
68
|
+
* An unknown type is passed through as a plain string rather than dropped:
|
|
69
|
+
* `Detection.entityType` is `EntityType | string` precisely so a client one
|
|
70
|
+
* release behind the service does not silently lose a whole category of PII.
|
|
71
|
+
*/
|
|
72
|
+
function toEntityType(raw) {
|
|
73
|
+
const canonical = (0, types_js_1.canonicalType)(raw);
|
|
74
|
+
const known = Object.values(types_js_1.EntityType).includes(canonical);
|
|
75
|
+
return known ? canonical : canonical;
|
|
76
|
+
}
|
|
77
|
+
function toResult(payload, text) {
|
|
78
|
+
const detections = (payload.entities ?? []).map(span => ({
|
|
79
|
+
entityType: toEntityType(span.type),
|
|
80
|
+
start: span.start,
|
|
81
|
+
end: span.end,
|
|
82
|
+
text: span.text,
|
|
83
|
+
source: span.source === "model" ? types_js_1.DetectionSource.CLOUD : types_js_1.DetectionSource.RULES,
|
|
84
|
+
country: null,
|
|
85
|
+
confidence: span.confidence ?? "high",
|
|
86
|
+
}));
|
|
87
|
+
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
88
|
+
return {
|
|
89
|
+
redactedText: payload.redacted_text ?? text,
|
|
90
|
+
detections,
|
|
91
|
+
source: "cloud",
|
|
92
|
+
degraded: false,
|
|
93
|
+
inferredCountries: [],
|
|
94
|
+
evidence: [],
|
|
95
|
+
detectionMode: "declared",
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
class CloudClient {
|
|
99
|
+
config;
|
|
100
|
+
constructor(config) {
|
|
101
|
+
const resolved = config ?? (0, config_js_1.getConfig)();
|
|
102
|
+
if (resolved === null)
|
|
103
|
+
throw new errors_js_1.NotConfiguredError();
|
|
104
|
+
this.config = resolved;
|
|
105
|
+
}
|
|
106
|
+
headers(idempotencyKey) {
|
|
107
|
+
return {
|
|
108
|
+
Authorization: `Bearer ${this.config.apiKey}`,
|
|
109
|
+
"Content-Type": "application/json",
|
|
110
|
+
// A retry after a timeout must not create a second job or a second usage
|
|
111
|
+
// row. The key is per logical document, not per attempt.
|
|
112
|
+
"Idempotency-Key": idempotencyKey,
|
|
113
|
+
...this.config.headers,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
raiseFor(status, payload) {
|
|
117
|
+
const message = payload.error ?? `HTTP ${status}`;
|
|
118
|
+
const detail = payload.detail ?? {};
|
|
119
|
+
if (status === 401)
|
|
120
|
+
throw new errors_js_1.CloudError(`authentication failed: ${message}`, status);
|
|
121
|
+
if (status === 413)
|
|
122
|
+
throw new errors_js_1.TooLargeError(message, status, detail);
|
|
123
|
+
if (status === 429)
|
|
124
|
+
throw new errors_js_1.QuotaExceededError(message, status, detail);
|
|
125
|
+
throw new errors_js_1.CloudError(message, status, detail);
|
|
126
|
+
}
|
|
127
|
+
async redact(text, options) {
|
|
128
|
+
const doFetch = requireFetch(options.fetchImpl);
|
|
129
|
+
const key = options.idempotencyKey ?? uuid();
|
|
130
|
+
const body = JSON.stringify({
|
|
131
|
+
text,
|
|
132
|
+
country: options.country.toUpperCase(),
|
|
133
|
+
language: options.language ?? "",
|
|
134
|
+
priority: options.priority ?? "interactive",
|
|
135
|
+
rules_only: options.rulesOnly ?? false,
|
|
136
|
+
});
|
|
137
|
+
let attempt = 0;
|
|
138
|
+
for (;;) {
|
|
139
|
+
let response;
|
|
140
|
+
let payload = {};
|
|
141
|
+
try {
|
|
142
|
+
response = await this.withTimeout(doFetch, `${this.config.baseUrl}/v1/redact`, {
|
|
143
|
+
method: "POST",
|
|
144
|
+
headers: this.headers(key),
|
|
145
|
+
body,
|
|
146
|
+
});
|
|
147
|
+
payload = await readJson(response);
|
|
148
|
+
}
|
|
149
|
+
catch (e) {
|
|
150
|
+
if (attempt >= this.config.maxRetries) {
|
|
151
|
+
throw new errors_js_1.CloudError(`cloud request failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (response) {
|
|
155
|
+
if (response.status === 202) {
|
|
156
|
+
// Outlived the sync window. Location is where the answer will appear;
|
|
157
|
+
// polling it is this client's job, not the caller's.
|
|
158
|
+
const location = response.headers.get("Location") ??
|
|
159
|
+
payload.location ??
|
|
160
|
+
`/v1/jobs/${payload.job_id ?? ""}`;
|
|
161
|
+
return toResult(await this.poll(doFetch, location, key), text);
|
|
162
|
+
}
|
|
163
|
+
if (response.status === 200)
|
|
164
|
+
return toResult(payload, text);
|
|
165
|
+
if (!RETRY_STATUS.has(response.status) || attempt >= this.config.maxRetries) {
|
|
166
|
+
this.raiseFor(response.status, payload);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
await sleep(backoffMs(attempt, retryAfterMs(response?.headers)));
|
|
170
|
+
attempt++;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
async poll(doFetch, location, key) {
|
|
174
|
+
const url = location.startsWith("http")
|
|
175
|
+
? location
|
|
176
|
+
: `${this.config.baseUrl}${location}`;
|
|
177
|
+
const deadline = Date.now() + this.config.pollTimeoutMs;
|
|
178
|
+
let delay = 500;
|
|
179
|
+
for (;;) {
|
|
180
|
+
const response = await this.withTimeout(doFetch, url, {
|
|
181
|
+
method: "GET",
|
|
182
|
+
headers: this.headers(key),
|
|
183
|
+
});
|
|
184
|
+
const payload = await readJson(response);
|
|
185
|
+
if (response.status === 200 && payload.redacted_text !== undefined) {
|
|
186
|
+
return payload;
|
|
187
|
+
}
|
|
188
|
+
if (response.status !== 200 && !RETRY_STATUS.has(response.status)) {
|
|
189
|
+
this.raiseFor(response.status, payload);
|
|
190
|
+
}
|
|
191
|
+
if (Date.now() >= deadline) {
|
|
192
|
+
throw new errors_js_1.CloudError(`document did not complete within ${Math.round(this.config.pollTimeoutMs / 1000)}s (${url})`);
|
|
193
|
+
}
|
|
194
|
+
const after = retryAfterMs(response.headers);
|
|
195
|
+
await sleep(after ?? delay);
|
|
196
|
+
delay = Math.min(delay * 1.5, 5000);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
async withTimeout(doFetch, url, init) {
|
|
200
|
+
const controller = new AbortController();
|
|
201
|
+
const timer = setTimeout(() => controller.abort(), this.config.timeoutMs);
|
|
202
|
+
try {
|
|
203
|
+
return await doFetch(url, { ...init, signal: controller.signal });
|
|
204
|
+
}
|
|
205
|
+
finally {
|
|
206
|
+
clearTimeout(timer);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
exports.CloudClient = CloudClient;
|
|
211
|
+
async function readJson(response) {
|
|
212
|
+
try {
|
|
213
|
+
const data = await response.json();
|
|
214
|
+
return data && typeof data === "object" ? data : {};
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return {};
|
|
218
|
+
}
|
|
219
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/** [CLOUD EXTENSION] Cloud tier configuration. */
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.DEFAULT_MAX_RETRIES = exports.DEFAULT_POLL_TIMEOUT_MS = exports.DEFAULT_TIMEOUT_MS = exports.DEFAULT_BASE_URL = void 0;
|
|
5
|
+
exports.configure = configure;
|
|
6
|
+
exports.getConfig = getConfig;
|
|
7
|
+
exports.reset = reset;
|
|
8
|
+
exports.DEFAULT_BASE_URL = "https://api.euredact.dev";
|
|
9
|
+
/**
|
|
10
|
+
* How long to wait for a single document. The service holds a connection open
|
|
11
|
+
* for its sync window and then hands back a job handle, so this has to exceed
|
|
12
|
+
* that window or the client abandons requests the service is about to answer.
|
|
13
|
+
*/
|
|
14
|
+
exports.DEFAULT_TIMEOUT_MS = 90_000;
|
|
15
|
+
/**
|
|
16
|
+
* Total wall-clock budget for a document that outlives the sync window and has
|
|
17
|
+
* to be polled. Cold-starting a GPU node and loading an ~18 GB checkpoint takes
|
|
18
|
+
* minutes, and an async job absorbs that invisibly — but not forever.
|
|
19
|
+
*/
|
|
20
|
+
exports.DEFAULT_POLL_TIMEOUT_MS = 900_000;
|
|
21
|
+
exports.DEFAULT_MAX_RETRIES = 3;
|
|
22
|
+
let config = null;
|
|
23
|
+
/** Read an env var where there is an environment to read. Browsers have none. */
|
|
24
|
+
function env(name) {
|
|
25
|
+
const proc = globalThis.process;
|
|
26
|
+
return proc?.env?.[name];
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Configure the cloud tier.
|
|
30
|
+
*
|
|
31
|
+
* Falls back to `EUREDACT_API_KEY` and `EUREDACT_BASE_URL` so a key never has
|
|
32
|
+
* to be written into source to get started.
|
|
33
|
+
*/
|
|
34
|
+
function configure(options = {}) {
|
|
35
|
+
const apiKey = options.apiKey ?? env("EUREDACT_API_KEY") ?? "";
|
|
36
|
+
if (!apiKey) {
|
|
37
|
+
throw new Error("no API key: pass configure({ apiKey }) or set EUREDACT_API_KEY");
|
|
38
|
+
}
|
|
39
|
+
config = {
|
|
40
|
+
apiKey,
|
|
41
|
+
baseUrl: (options.baseUrl ?? env("EUREDACT_BASE_URL") ?? exports.DEFAULT_BASE_URL)
|
|
42
|
+
.replace(/\/+$/, ""),
|
|
43
|
+
timeoutMs: options.timeoutMs ?? exports.DEFAULT_TIMEOUT_MS,
|
|
44
|
+
pollTimeoutMs: options.pollTimeoutMs ?? exports.DEFAULT_POLL_TIMEOUT_MS,
|
|
45
|
+
maxRetries: options.maxRetries ?? exports.DEFAULT_MAX_RETRIES,
|
|
46
|
+
headers: { ...(options.headers ?? {}) },
|
|
47
|
+
};
|
|
48
|
+
return config;
|
|
49
|
+
}
|
|
50
|
+
/** The active configuration, or null. Never throws. */
|
|
51
|
+
function getConfig() {
|
|
52
|
+
return config;
|
|
53
|
+
}
|
|
54
|
+
/** Forget the configuration. Mainly for tests. */
|
|
55
|
+
function reset() {
|
|
56
|
+
config = null;
|
|
57
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/** [CLOUD EXTENSION] Errors the cloud tier can raise. */
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.TooLargeError = exports.QuotaExceededError = exports.NotConfiguredError = exports.CloudError = void 0;
|
|
5
|
+
class CloudError extends Error {
|
|
6
|
+
status;
|
|
7
|
+
detail;
|
|
8
|
+
constructor(message, status, detail) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "CloudError";
|
|
11
|
+
this.status = status;
|
|
12
|
+
this.detail = detail ?? {};
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
exports.CloudError = CloudError;
|
|
16
|
+
/**
|
|
17
|
+
* Raised when the cloud tier is used without configuration.
|
|
18
|
+
*
|
|
19
|
+
* Deliberately an error rather than a quiet fallback to rules-only output.
|
|
20
|
+
* `mode: "cloud"` that silently returns rules-only results is the worst failure
|
|
21
|
+
* this library can have: the caller believes names, employers and diagnoses
|
|
22
|
+
* were checked, sees a plausible redacted document, and ships it with the PII
|
|
23
|
+
* still in it.
|
|
24
|
+
*/
|
|
25
|
+
class NotConfiguredError extends CloudError {
|
|
26
|
+
constructor(message) {
|
|
27
|
+
super(message ??
|
|
28
|
+
"Cloud tier not configured. Call euredact.configure({ apiKey }) first, " +
|
|
29
|
+
"or set EUREDACT_API_KEY.");
|
|
30
|
+
this.name = "NotConfiguredError";
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
exports.NotConfiguredError = NotConfiguredError;
|
|
34
|
+
/** 429 after retries were exhausted. */
|
|
35
|
+
class QuotaExceededError extends CloudError {
|
|
36
|
+
constructor(message, status, detail) {
|
|
37
|
+
super(message, status, detail);
|
|
38
|
+
this.name = "QuotaExceededError";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
exports.QuotaExceededError = QuotaExceededError;
|
|
42
|
+
/**
|
|
43
|
+
* 413. The document is over the service's input cap.
|
|
44
|
+
*
|
|
45
|
+
* Permanent and not retryable. The model has never seen a chunk boundary, so
|
|
46
|
+
* the service refuses oversized input rather than splitting it and changing the
|
|
47
|
+
* accuracy story silently.
|
|
48
|
+
*/
|
|
49
|
+
class TooLargeError extends CloudError {
|
|
50
|
+
constructor(message, status, detail) {
|
|
51
|
+
super(message, status, detail);
|
|
52
|
+
this.name = "TooLargeError";
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
exports.TooLargeError = TooLargeError;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* [CLOUD EXTENSION] Cloud tier — rules engine plus a fine-tuned model.
|
|
4
|
+
*
|
|
5
|
+
* The rule engine catches what has a shape: IBANs, national IDs, phone
|
|
6
|
+
* numbers, anything with a checksum. It cannot catch what does not — a
|
|
7
|
+
* person's name, an employer, a diagnosis, a job title. Those are what the
|
|
8
|
+
* cloud tier adds, and the `EntityType` members marked `[CLOUD EXTENSION]` are
|
|
9
|
+
* exactly that gap.
|
|
10
|
+
*
|
|
11
|
+
* ```ts
|
|
12
|
+
* import { configure, redactAsync } from "euredact";
|
|
13
|
+
*
|
|
14
|
+
* configure({ apiKey: "erk_..." });
|
|
15
|
+
* const result = await redactAsync(text, { countries: ["BE"], mode: "cloud" });
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.TooLargeError = exports.QuotaExceededError = exports.NotConfiguredError = exports.CloudError = exports.DEFAULT_BASE_URL = exports.reset = exports.getConfig = exports.configure = exports.CloudClient = void 0;
|
|
20
|
+
var client_js_1 = require("./client.js");
|
|
21
|
+
Object.defineProperty(exports, "CloudClient", { enumerable: true, get: function () { return client_js_1.CloudClient; } });
|
|
22
|
+
var config_js_1 = require("./config.js");
|
|
23
|
+
Object.defineProperty(exports, "configure", { enumerable: true, get: function () { return config_js_1.configure; } });
|
|
24
|
+
Object.defineProperty(exports, "getConfig", { enumerable: true, get: function () { return config_js_1.getConfig; } });
|
|
25
|
+
Object.defineProperty(exports, "reset", { enumerable: true, get: function () { return config_js_1.reset; } });
|
|
26
|
+
Object.defineProperty(exports, "DEFAULT_BASE_URL", { enumerable: true, get: function () { return config_js_1.DEFAULT_BASE_URL; } });
|
|
27
|
+
var errors_js_1 = require("./errors.js");
|
|
28
|
+
Object.defineProperty(exports, "CloudError", { enumerable: true, get: function () { return errors_js_1.CloudError; } });
|
|
29
|
+
Object.defineProperty(exports, "NotConfiguredError", { enumerable: true, get: function () { return errors_js_1.NotConfiguredError; } });
|
|
30
|
+
Object.defineProperty(exports, "QuotaExceededError", { enumerable: true, get: function () { return errors_js_1.QuotaExceededError; } });
|
|
31
|
+
Object.defineProperty(exports, "TooLargeError", { enumerable: true, get: function () { return errors_js_1.TooLargeError; } });
|
package/dist/cjs/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SEED_BIC6_PREFIXES = exports.getBicRegistry = exports.setBicRegistry = exports.COUNTRY_CONFIGS = exports.EuRedact = exports.DocumentContext = exports.LEGACY_TYPE_ALIASES = exports.DetectionSource = exports.EntityType = void 0;
|
|
3
|
+
exports.TooLargeError = exports.QuotaExceededError = exports.NotConfiguredError = exports.CloudError = exports.CloudClient = exports.getConfig = exports.configure = exports.SEED_BIC6_PREFIXES = exports.getBicRegistry = exports.setBicRegistry = exports.COUNTRY_CONFIGS = exports.EuRedact = exports.DocumentContext = exports.canonicalType = exports.LEGACY_TYPE_ALIASES = exports.DetectionSource = exports.EntityType = void 0;
|
|
4
4
|
exports.availableCountries = availableCountries;
|
|
5
5
|
exports.redact = redact;
|
|
6
|
+
exports.redactAsync = redactAsync;
|
|
6
7
|
exports.addCustomPattern = addCustomPattern;
|
|
7
8
|
exports.clear = clear;
|
|
8
9
|
exports.redactBatch = redactBatch;
|
|
@@ -10,6 +11,7 @@ var types_js_1 = require("./types.js");
|
|
|
10
11
|
Object.defineProperty(exports, "EntityType", { enumerable: true, get: function () { return types_js_1.EntityType; } });
|
|
11
12
|
Object.defineProperty(exports, "DetectionSource", { enumerable: true, get: function () { return types_js_1.DetectionSource; } });
|
|
12
13
|
Object.defineProperty(exports, "LEGACY_TYPE_ALIASES", { enumerable: true, get: function () { return types_js_1.LEGACY_TYPE_ALIASES; } });
|
|
14
|
+
Object.defineProperty(exports, "canonicalType", { enumerable: true, get: function () { return types_js_1.canonicalType; } });
|
|
13
15
|
var context_js_1 = require("./rules/context.js");
|
|
14
16
|
Object.defineProperty(exports, "DocumentContext", { enumerable: true, get: function () { return context_js_1.DocumentContext; } });
|
|
15
17
|
var sdk_js_1 = require("./sdk.js");
|
|
@@ -20,8 +22,16 @@ var bicRegistry_js_1 = require("./rules/bicRegistry.js");
|
|
|
20
22
|
Object.defineProperty(exports, "setBicRegistry", { enumerable: true, get: function () { return bicRegistry_js_1.setBicRegistry; } });
|
|
21
23
|
Object.defineProperty(exports, "getBicRegistry", { enumerable: true, get: function () { return bicRegistry_js_1.getBicRegistry; } });
|
|
22
24
|
Object.defineProperty(exports, "SEED_BIC6_PREFIXES", { enumerable: true, get: function () { return bicRegistry_js_1.SEED_BIC6_PREFIXES; } });
|
|
25
|
+
var index_js_2 = require("./cloud/index.js");
|
|
26
|
+
Object.defineProperty(exports, "configure", { enumerable: true, get: function () { return index_js_2.configure; } });
|
|
27
|
+
Object.defineProperty(exports, "getConfig", { enumerable: true, get: function () { return index_js_2.getConfig; } });
|
|
28
|
+
Object.defineProperty(exports, "CloudClient", { enumerable: true, get: function () { return index_js_2.CloudClient; } });
|
|
29
|
+
Object.defineProperty(exports, "CloudError", { enumerable: true, get: function () { return index_js_2.CloudError; } });
|
|
30
|
+
Object.defineProperty(exports, "NotConfiguredError", { enumerable: true, get: function () { return index_js_2.NotConfiguredError; } });
|
|
31
|
+
Object.defineProperty(exports, "QuotaExceededError", { enumerable: true, get: function () { return index_js_2.QuotaExceededError; } });
|
|
32
|
+
Object.defineProperty(exports, "TooLargeError", { enumerable: true, get: function () { return index_js_2.TooLargeError; } });
|
|
23
33
|
const sdk_js_2 = require("./sdk.js");
|
|
24
|
-
const
|
|
34
|
+
const index_js_3 = require("./rules/countries/index.js");
|
|
25
35
|
let _instance = null;
|
|
26
36
|
function getInstance() {
|
|
27
37
|
if (_instance === null)
|
|
@@ -29,11 +39,20 @@ function getInstance() {
|
|
|
29
39
|
return _instance;
|
|
30
40
|
}
|
|
31
41
|
function availableCountries() {
|
|
32
|
-
return Object.keys(
|
|
42
|
+
return Object.keys(index_js_3.COUNTRY_CONFIGS).filter(c => c !== "SHARED").sort();
|
|
33
43
|
}
|
|
34
44
|
function redact(text, options) {
|
|
35
45
|
return getInstance().redact(text, options);
|
|
36
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* Redact, awaiting the cloud tier when `mode: "cloud"` is asked for.
|
|
49
|
+
*
|
|
50
|
+
* `mode: "rules"` resolves immediately with exactly what `redact()` returns,
|
|
51
|
+
* so a caller that may or may not use the cloud tier can hold one code path.
|
|
52
|
+
*/
|
|
53
|
+
function redactAsync(text, options) {
|
|
54
|
+
return getInstance().redactAsync(text, options);
|
|
55
|
+
}
|
|
37
56
|
function addCustomPattern(name, pattern) {
|
|
38
57
|
getInstance().addCustomPattern(name, pattern);
|
|
39
58
|
}
|