euredact 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +403 -39
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +124 -14
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +63 -10
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +292 -22
- package/dist/cjs/rules/validators.js +50 -0
- package/dist/cjs/sdk.js +18 -7
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +83 -1
- package/dist/esm/cloud/client.js +119 -22
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +63 -10
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +291 -21
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +47 -0
- package/dist/esm/sdk.d.ts +51 -4
- package/dist/esm/sdk.js +18 -9
- package/dist/esm/types.d.ts +58 -2
- package/dist/esm/types.js +4 -1
- package/package.json +12 -4
package/NOTICE
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
euredact
|
|
2
|
+
Copyright the euredact authors. Licensed under the Apache License 2.0;
|
|
3
|
+
see LICENSE.
|
|
4
|
+
|
|
5
|
+
This package bundles BIC6 prefixes derived from the BIC/LEI Mapping Table,
|
|
6
|
+
published by GLEIF (https://www.gleif.org) and developed by SWIFT, used
|
|
7
|
+
under the BIC/LEI Mapping Table License Agreement:
|
|
8
|
+
|
|
9
|
+
SWIFT © and database rights September 2026. All rights reserved.
|
|
10
|
+
This Mapping Table has been developed by SWIFT. Any use of the Mapping Table, in whole or in part, is subject to the BIC/LEI Mapping Table License Agreement as published with the Mapping Table available on GLEIF's website.
|
|
11
|
+
The Mapping Table is updated monthly. For the latest BIC information and updates, always refer to www.swift.com/bic .
|
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ console.log(result.detections);
|
|
|
34
34
|
## Features
|
|
35
35
|
|
|
36
36
|
- **31 European countries** with country-specific patterns
|
|
37
|
-
- **
|
|
37
|
+
- **40 PII entity types** (30 from the rules engine, 10 from the
|
|
38
38
|
[cloud tier](#cloud-tier)): national IDs, IBANs, phone numbers, email, VAT
|
|
39
39
|
numbers, license plates, credit cards, BIC/SWIFT, VIN, IMEI, GPS coordinates,
|
|
40
40
|
UUIDs, social handles, IP/IPv6, MAC addresses, secrets/API keys, and more
|
|
@@ -114,6 +114,7 @@ interface RedactOptions {
|
|
|
114
114
|
allowlistDomains?: string[] | null; // domains never redacted (EMAIL/URL)
|
|
115
115
|
detectDates?: boolean; // include DOB / date of death
|
|
116
116
|
cache?: boolean; // reuse results for identical input
|
|
117
|
+
idempotencyKey?: string; // mode: "cloud" only; see the table below
|
|
117
118
|
}
|
|
118
119
|
```
|
|
119
120
|
|
|
@@ -141,6 +142,7 @@ interface RedactOptions {
|
|
|
141
142
|
| Parameter | Default | Description |
|
|
142
143
|
|---|---|---|
|
|
143
144
|
| `mode` | `"rules"` | `"rules"` runs locally and synchronously. `"cloud"` must go through [`redactAsync()`](#redactasynctext-options) — `redact()` is synchronous and a network call cannot be. See [Cloud tier](#cloud-tier). |
|
|
145
|
+
| `idempotencyKey` | generated | `mode: "cloud"` only (through `redactAsync`): sent as the request's `Idempotency-Key`, so the same key returns the same job instead of running and billing the document again. Throws in rules mode. See [What the service said](#what-the-service-said-resultcloud). |
|
|
144
146
|
| `context` | `null` | Share country evidence across the chunks of one document. Pass the same `DocumentContext` to every chunk. Disables the cache. See [Chunked documents](#chunked-documents). |
|
|
145
147
|
| `chunkOffset` | `0` | Where this chunk starts in the whole document. Only rebases spans recorded in `context`; returned detections stay relative to `text`. |
|
|
146
148
|
| `cache` | `true` | Reuse the result for an identical input and configuration. |
|
|
@@ -248,10 +250,18 @@ instance, which is what makes one instance per tenant the right default.
|
|
|
248
250
|
interface RedactResult {
|
|
249
251
|
redactedText: string; // The input text with PII replaced
|
|
250
252
|
detections: Detection[]; // All PII spans found
|
|
251
|
-
source: string; //
|
|
253
|
+
source: string; // "rules" or "cloud"
|
|
252
254
|
degraded: boolean; // True if the engine fell back to a simpler mode
|
|
255
|
+
|
|
256
|
+
// Country inference — see "Country-independent detection"
|
|
257
|
+
inferredCountries: Array<[string, number]>; // [country, confidence], strongest first
|
|
258
|
+
evidence: CountryEvidence[]; // every signal, with the span behind it
|
|
259
|
+
detectionMode: string; // "declared" if countries was passed,
|
|
260
|
+
// "inferred" otherwise
|
|
253
261
|
tokens: Record<string, string>; // token -> original value; only with tokenize: true
|
|
254
262
|
exempted: Exemption[]; // spans the allowlist kept, with the rule that matched
|
|
263
|
+
cloud?: CloudInfo; // mode: "cloud" only: job id, model, cost, unplaced
|
|
264
|
+
// findings — see "What the service said"
|
|
255
265
|
}
|
|
256
266
|
```
|
|
257
267
|
|
|
@@ -265,7 +275,7 @@ interface Detection {
|
|
|
265
275
|
text: string; // The matched substring
|
|
266
276
|
source: DetectionSource; // "rules" or "cloud"
|
|
267
277
|
country: string | null; // ISO code or null for shared/custom patterns
|
|
268
|
-
confidence: string;
|
|
278
|
+
confidence: string | null; // "high" | "medium" | "low"; null for a cloud-model span — see below
|
|
269
279
|
countryConfidence?: number; // How strongly the document supports `country`,
|
|
270
280
|
// in [0, 1]. 0 means the attribution rests on
|
|
271
281
|
// a checksum alone.
|
|
@@ -282,6 +292,7 @@ about whether the span is masked — every detection is, at every level.
|
|
|
282
292
|
| `"high"` | a pattern matched and, where one exists, its checksum passed |
|
|
283
293
|
| `"medium"` | the type comes from a label touching the span, because no pattern of that type claimed it — `Αρ. Ταυτότητας: 00892341` is a `NATIONAL_ID` although nothing can checksum it |
|
|
284
294
|
| `"low"` | a pattern matched, its checksum *failed*, and the document labels the span as that very type — `Rijksregisternummer: 85.03.19-284.73` is a national number with a bad check digit |
|
|
295
|
+
| `null` | a span the cloud model found (`source` is `cloud`). The model reports no confidence, so none is invented: a model span is not checksum-backed, and filtering on `"high"` keeps rules detections only *(rules-engine#108)* |
|
|
285
296
|
|
|
286
297
|
Filter on it when you need only checksum-backed detections:
|
|
287
298
|
|
|
@@ -294,24 +305,6 @@ identifier: the shape and the label agree, the check digit does not. Earlier
|
|
|
294
305
|
versions dropped these, which meant a redaction library printed in full an
|
|
295
306
|
identifier it had recognised and rejected.
|
|
296
307
|
|
|
297
|
-
#### `RedactResult`
|
|
298
|
-
|
|
299
|
-
```ts
|
|
300
|
-
interface RedactResult {
|
|
301
|
-
redactedText: string;
|
|
302
|
-
detections: Detection[];
|
|
303
|
-
source: string;
|
|
304
|
-
degraded: boolean;
|
|
305
|
-
|
|
306
|
-
// Country inference — see "Country-independent detection"
|
|
307
|
-
inferredCountries: Array<[string, number]>; // [country, confidence], strongest first
|
|
308
|
-
evidence: CountryEvidence[]; // every signal, with the span behind it
|
|
309
|
-
detectionMode: string; // "declared" if countries was passed,
|
|
310
|
-
// "inferred" otherwise
|
|
311
|
-
tokens: Record<string, string>; // token -> original value; only with tokenize: true
|
|
312
|
-
}
|
|
313
|
-
```
|
|
314
|
-
|
|
315
308
|
#### `EntityType`
|
|
316
309
|
|
|
317
310
|
String enum with all supported PII categories:
|
|
@@ -330,10 +323,15 @@ Cloud tier only — the rule engine never emits these, because there is no shape
|
|
|
330
323
|
to match on. That is precisely why the model exists:
|
|
331
324
|
|
|
332
325
|
ORGANISATION_NAME JOB_TITLE MEDICAL_CONDITION SENSITIVE_ATTRIBUTE
|
|
333
|
-
BIOMETRIC_REF FINANCIAL_AMOUNT QUASI_IDENTIFIER
|
|
334
|
-
URL
|
|
326
|
+
BIOMETRIC_REF FINANCIAL_AMOUNT QUASI_IDENTIFIER URL
|
|
335
327
|
```
|
|
336
328
|
|
|
329
|
+
`SECRET` is the one type for credentials, whichever layer finds them: the rules
|
|
330
|
+
engine's shaped keys and tokens, and the cloud model's prose passwords and
|
|
331
|
+
connection-string secrets, which the service labels `CREDENTIAL`. The SDK
|
|
332
|
+
reports those as `SECRET`, and `CREDENTIAL` remains only as a legacy alias of
|
|
333
|
+
it (rules-engine#111).
|
|
334
|
+
|
|
337
335
|
`INTERNAL_ID` — an employee, badge or customer number tied to a person — is
|
|
338
336
|
emitted **only** when an explicit label names it (`medarbejdernummer:`,
|
|
339
337
|
`Personalnummer:`, `Employee No:`, `Betriebsstättennr.`, `Badge`). There is no
|
|
@@ -420,8 +418,10 @@ const results = await Promise.all(
|
|
|
420
418
|
);
|
|
421
419
|
```
|
|
422
420
|
|
|
423
|
-
Bound that yourself if the batch is large
|
|
424
|
-
|
|
421
|
+
Bound that yourself if the batch is large. The service's rate limit answers
|
|
422
|
+
`429`, which the client retries, honouring `Retry-After`, before throwing
|
|
423
|
+
`RateLimitedError`. A `429` for the daily quota throws `QuotaExceededError`
|
|
424
|
+
at once: it will not clear before the day does.
|
|
425
425
|
|
|
426
426
|
### Reuse one instance
|
|
427
427
|
|
|
@@ -473,6 +473,11 @@ result.detections.map(d => [d.entityType, d.text]);
|
|
|
473
473
|
// [["PERSON_NAME", "Bas Verhoeven"], ["PHONE", "+32 ..."]]
|
|
474
474
|
```
|
|
475
475
|
|
|
476
|
+
The endpoint defaults to `https://api.euredact.dev`; `baseUrl` (or
|
|
477
|
+
`EUREDACT_BASE_URL`) overrides it. It must be `https://`: plain `http://` is
|
|
478
|
+
refused except to `localhost`, `127.0.0.1` or `::1`, so the key and the text
|
|
479
|
+
never travel unencrypted.
|
|
480
|
+
|
|
476
481
|
**`redact()` is synchronous and cannot serve the cloud tier**, so
|
|
477
482
|
`redact(text, { mode: "cloud" })` throws and names `redactAsync`. It never
|
|
478
483
|
falls back to rules-only output: a caller who believes names and diagnoses were
|
|
@@ -590,6 +595,340 @@ either, no hosted model can help; run one yourself against
|
|
|
590
595
|
`local.redactedText`.
|
|
591
596
|
|
|
592
597
|
|
|
598
|
+
### What the service said: `result.cloud`
|
|
599
|
+
|
|
600
|
+
A cloud result carries `cloud`, what the service reported beside the spans;
|
|
601
|
+
it is absent on a rules-only result.
|
|
602
|
+
|
|
603
|
+
```ts
|
|
604
|
+
const result = await redactAsync(text, {
|
|
605
|
+
countries: ["BE"], mode: "cloud", idempotencyKey: "invoice-2291-v1",
|
|
606
|
+
});
|
|
607
|
+
result.cloud?.jobId; // support, Jobs.retrieve, replays
|
|
608
|
+
result.cloud?.modelVersion; // the model that answered; null when only rules ran
|
|
609
|
+
for (const item of result.cloud?.unlocated ?? []) console.log(item.entityType, item.text);
|
|
610
|
+
const usage = result.cloud?.usage;
|
|
611
|
+
if (usage) {
|
|
612
|
+
const { tokens, billingRate, credits, factors } = usage;
|
|
613
|
+
for (const f of factors) console.log(f.code, f.types ?? "", f.detail);
|
|
614
|
+
}
|
|
615
|
+
```
|
|
616
|
+
|
|
617
|
+
- **`idempotencyKey`** is sent as the request's `Idempotency-Key`: the same key
|
|
618
|
+
returns the same job instead of running and billing the document again. One
|
|
619
|
+
is generated per call when omitted. It applies to `mode: "cloud"` only.
|
|
620
|
+
- **`usage`** is the total the request was charged and the reasons for it, never
|
|
621
|
+
a cost per step. `credits` is `round(tokens × billingRate)`, the formula the
|
|
622
|
+
service debits by. The factor codes are `rules_only`, `document_length`,
|
|
623
|
+
`instructions`, `dense_document`, `special_category_check` (with the `types`
|
|
624
|
+
that triggered it), `long_document` and `batch_rate`; a code added later
|
|
625
|
+
still arrives, as its string. Absent when the service does not report it or
|
|
626
|
+
reported it malformed: a cost report never fails a redaction.
|
|
627
|
+
- **`unlocated`** lists what the model found but could not match to a position
|
|
628
|
+
in the text it was sent. Nothing was masked for these.
|
|
629
|
+
- **Batch documents** carry `modelVersion` and `unlocated`; no `jobId` or
|
|
630
|
+
`usage`, which belong to the batch (`batch.tokens`, `batch.creditsCharged`,
|
|
631
|
+
`batch.billingRate`).
|
|
632
|
+
|
|
633
|
+
All four types are exported from `euredact`:
|
|
634
|
+
|
|
635
|
+
```ts
|
|
636
|
+
interface CloudInfo {
|
|
637
|
+
jobId: string | null; // null for a batch document
|
|
638
|
+
modelVersion: string | null; // null when only rules ran
|
|
639
|
+
usage?: Usage; // absent from older services, batch documents, or a malformed block
|
|
640
|
+
unlocated: Unlocated[];
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
interface Unlocated {
|
|
644
|
+
text: string; // what the model reported
|
|
645
|
+
entityType: EntityType | string; // e.g. "PERSON_NAME"; unknown types stay strings
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
interface Usage {
|
|
649
|
+
tokens: number; // every model call the document triggered
|
|
650
|
+
billingRate: number; // 1, or e.g. 0.75 for a batch
|
|
651
|
+
credits: number; // round(tokens * billingRate): what was debited
|
|
652
|
+
factors: UsageFactor[];
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
interface UsageFactor {
|
|
656
|
+
code: string; // see the list above; new codes arrive as strings
|
|
657
|
+
detail: string; // one sentence from the service ("" if none)
|
|
658
|
+
types?: string[]; // special_category_check only
|
|
659
|
+
}
|
|
660
|
+
```
|
|
661
|
+
|
|
662
|
+
**Retrying safely.** Pass your own `idempotencyKey` when your code may send the
|
|
663
|
+
same document twice: after a timeout of your own, from a retried queue message
|
|
664
|
+
or a restarted worker. Derive it from something stable (an invoice number plus
|
|
665
|
+
a version), not from a random value per attempt. A replay returns the same
|
|
666
|
+
`result.cloud.jobId` and is not billed again.
|
|
667
|
+
|
|
668
|
+
### Account, past jobs and the batch list
|
|
669
|
+
|
|
670
|
+
Everything an API key may do on the service is reachable from the SDK. What
|
|
671
|
+
needs a signed-in person (logging in, minting keys, settings, members, terms)
|
|
672
|
+
is not, because a key cannot do it. Every call returns a `Promise`.
|
|
673
|
+
|
|
674
|
+
| Call | Service endpoint | Resolves to |
|
|
675
|
+
|---|---|---|
|
|
676
|
+
| `new Account().summary()` | `GET /v1/account` | `AccountSummary` |
|
|
677
|
+
| `new Account().credits()` | `GET /v1/account/credits` | `Credits` |
|
|
678
|
+
| `new Account().creditHistory(limit = 50)` | `GET /v1/account/credits/history` | `CreditEntry[]`, newest first; `limit` 1–500 |
|
|
679
|
+
| `new Account().usage(days = 30)` | `GET /v1/account/usage` | `UsageDay[]` |
|
|
680
|
+
| `new Account().usageByKey(days = 30)` | `GET /v1/account/usage/by-key` | `KeyUsage[]` |
|
|
681
|
+
| `new Account().keys()` | `GET /v1/account/keys` | `ApiKey[]` |
|
|
682
|
+
| `new Account().revokeKey(keyId)` | `POST /v1/account/keys/{id}/revoke` | `KeyRevocation` |
|
|
683
|
+
| `new Jobs().retrieve(jobId)` | `GET /v1/jobs/{id}` | `Job` |
|
|
684
|
+
| `new Batches().list(limit = 20)` | `GET /v1/batches` | `Batch[]`, newest first; `limit` 1–100 |
|
|
685
|
+
|
|
686
|
+
```ts
|
|
687
|
+
import { Account, Batches, Jobs } from "euredact";
|
|
688
|
+
|
|
689
|
+
const account = new Account();
|
|
690
|
+
const summary = await account.summary();
|
|
691
|
+
if (summary.quotaRemaining !== null && summary.quotaRemaining < 100) {
|
|
692
|
+
console.log(`${summary.quotaRemaining} documents left today`);
|
|
693
|
+
}
|
|
694
|
+
console.log((await account.credits()).balance);
|
|
695
|
+
for (const day of await account.usage(7)) console.log(day.day, day.documents, day.tokens);
|
|
696
|
+
for (const key of await account.keys()) console.log(key.id, key.name, key.active, key.lastUsedAt);
|
|
697
|
+
|
|
698
|
+
const job = await new Jobs().retrieve("job-7f3a…");
|
|
699
|
+
if (job.result) console.log(job.result.redactedText);
|
|
700
|
+
|
|
701
|
+
for (const batch of await new Batches().list(20)) {
|
|
702
|
+
console.log(batch.id, batch.status, batch.documents, batch.creditsCharged);
|
|
703
|
+
}
|
|
704
|
+
```
|
|
705
|
+
|
|
706
|
+
#### Creating the clients
|
|
707
|
+
|
|
708
|
+
`Account` and `Jobs` read the configuration `configure()` set (or
|
|
709
|
+
`EUREDACT_API_KEY`). Both take two optional arguments:
|
|
710
|
+
`new Account(config?, { fetchImpl? })`. `config` is a `CloudConfig` of its own,
|
|
711
|
+
for a second key or another endpoint: a plain object with `apiKey`, `baseUrl`,
|
|
712
|
+
`timeoutMs`, `pollTimeoutMs`, `maxRetries` and `headers`, checked for an
|
|
713
|
+
`https://` base URL like `configure()` does; `fetchImpl` replaces the platform `fetch`,
|
|
714
|
+
for a proxy or a mock in your tests. (`Batches` takes the same two inside its
|
|
715
|
+
options object: `new Batches({ config, fetchImpl })`.) There is nothing to close.
|
|
716
|
+
They need a global `fetch`: Node 18+, or a browser, or pass `fetchImpl`.
|
|
717
|
+
|
|
718
|
+
#### What each call resolves to
|
|
719
|
+
|
|
720
|
+
Each value has `raw`, the service's JSON as received, so a field the SDK does
|
|
721
|
+
not type yet is still reachable; every field is `null` when the service left
|
|
722
|
+
it out or sent the wrong type, rather than the call failing.
|
|
723
|
+
|
|
724
|
+
```ts
|
|
725
|
+
interface AccountSummary {
|
|
726
|
+
tenantId: string | null; tenantName: string | null; active: boolean | null;
|
|
727
|
+
memberSince: string | null; // ISO date
|
|
728
|
+
role: string | null;
|
|
729
|
+
retentionMode: string | null; // "ttl", or "none": text discarded on delivery
|
|
730
|
+
payloadTtlHours: number | null; // 0 under "none"
|
|
731
|
+
documentsToday: number | null; documentsThisMonth: number | null;
|
|
732
|
+
tokensThisMonth: number | null; failures24h: number | null;
|
|
733
|
+
dailyQuota: number | null; // null: no daily quota on this account
|
|
734
|
+
quotaRemaining: number | null;
|
|
735
|
+
activeKeys: number | null;
|
|
736
|
+
batchesAvailable: boolean | null; // batches need retention ("ttl")
|
|
737
|
+
raw: Record<string, unknown>;
|
|
738
|
+
}
|
|
739
|
+
interface Credits { balance: number | null; granted: number | null; spent: number | null;
|
|
740
|
+
unit: string | null /* "tokens" */; raw: Record<string, unknown> }
|
|
741
|
+
interface CreditEntry { // rolled up per minute, reason and direction
|
|
742
|
+
at: string | null; delta: number | null /* negative for a debit */;
|
|
743
|
+
reason: string | null; count: number | null; jobId: string | null; sourceRef: string | null;
|
|
744
|
+
raw: Record<string, unknown>;
|
|
745
|
+
}
|
|
746
|
+
interface UsageDay { day: string | null; documents: number | null; tokens: number | null;
|
|
747
|
+
failures: number | null; rulesOnly: number | null; raw: Record<string, unknown> }
|
|
748
|
+
interface KeyUsage { // the busiest keys one by one; the rest in one entry, keyId null
|
|
749
|
+
keyId: number | null; name: string | null;
|
|
750
|
+
documents: number | null; tokens: number | null; failures: number | null;
|
|
751
|
+
series: UsageDay[]; raw: Record<string, unknown>;
|
|
752
|
+
}
|
|
753
|
+
interface ApiKey { // never the secret: shown once, when minted
|
|
754
|
+
id: number | null; name: string | null; createdBy: string | null;
|
|
755
|
+
createdAt: string | null; lastUsedAt: string | null; revokedAt: string | null;
|
|
756
|
+
active: boolean | null; raw: Record<string, unknown>;
|
|
757
|
+
}
|
|
758
|
+
interface KeyRevocation {
|
|
759
|
+
id: number | null; keysRemaining: number | null;
|
|
760
|
+
lockedOut: boolean | null; // true: no active key left; mint one in the console
|
|
761
|
+
selfRevoked: boolean | null; // true: you revoked the key this client uses
|
|
762
|
+
raw: Record<string, unknown>;
|
|
763
|
+
}
|
|
764
|
+
interface Job {
|
|
765
|
+
jobId: string;
|
|
766
|
+
status: string; // "queued", "running" or "succeeded"
|
|
767
|
+
createdAt: string | null;
|
|
768
|
+
result: RedactResult | null; // set once succeeded
|
|
769
|
+
raw: Record<string, unknown>;
|
|
770
|
+
}
|
|
771
|
+
```
|
|
772
|
+
|
|
773
|
+
`Batch` gained four fields beside `id`, `status`, `counts` and the timestamps:
|
|
774
|
+
`documents`, `tokens` (`{ prompt, completion }`), `creditsCharged` and
|
|
775
|
+
`billingRate` (e.g. `0.75`).
|
|
776
|
+
|
|
777
|
+
#### Retrieving a job
|
|
778
|
+
|
|
779
|
+
`redactAsync(text, { mode: "cloud" })` already waits for its job, polling past
|
|
780
|
+
the service's synchronous window, so most code never needs `Jobs`. It is for
|
|
781
|
+
support and audits (look up `result.cloud.jobId` later) and for a job the
|
|
782
|
+
client gave up polling on (`pollTimeoutMs`), which may still finish.
|
|
783
|
+
|
|
784
|
+
A failed or rejected job throws (`CloudError` with status 502, `TooLargeError`)
|
|
785
|
+
rather than resolving with a status. The result is over the text the service
|
|
786
|
+
received: under `mode: "cloud"` that is the locally masked text, so its offsets
|
|
787
|
+
and `redactedText` are not your original's. Results are kept for the account's
|
|
788
|
+
retention period (`AccountSummary.payloadTtlHours`), and not at all under
|
|
789
|
+
`retentionMode === "none"`; after that, `ResultExpiredError`.
|
|
790
|
+
|
|
791
|
+
#### Revoking a key
|
|
792
|
+
|
|
793
|
+
`revokeKey` is the one write an API key may make, for a key that leaked. It is
|
|
794
|
+
sent once and never retried. Revoking the key the client itself uses succeeds
|
|
795
|
+
and reports `selfRevoked: true`; every later call with it fails with status
|
|
796
|
+
401. When `lockedOut` is `true`, no active key is left and only a person in the
|
|
797
|
+
console can mint one.
|
|
798
|
+
|
|
799
|
+
#### Handling errors
|
|
800
|
+
|
|
801
|
+
All errors extend `CloudError`, which carries `status` (the HTTP status, or
|
|
802
|
+
`undefined` for a network failure) and `detail` (the service's JSON detail).
|
|
803
|
+
Test the specific ones first: `RateLimitedError` extends `QuotaExceededError`.
|
|
804
|
+
|
|
805
|
+
```ts
|
|
806
|
+
import {
|
|
807
|
+
CloudError, QuotaExceededError, RateLimitedError, TooLargeError, redactAsync,
|
|
808
|
+
} from "euredact";
|
|
809
|
+
|
|
810
|
+
try {
|
|
811
|
+
const result = await redactAsync(text, {
|
|
812
|
+
countries: ["BE"], mode: "cloud", idempotencyKey: `invoice-${invoice.id}-v${invoice.version}`,
|
|
813
|
+
});
|
|
814
|
+
} catch (err) {
|
|
815
|
+
if (err instanceof RateLimitedError) {
|
|
816
|
+
// rate limit still hit after every retry: back off, then retry with the same key
|
|
817
|
+
} else if (err instanceof QuotaExceededError) {
|
|
818
|
+
// daily quota used up (err.detail.used, err.detail.limit): wait for tomorrow
|
|
819
|
+
} else if (err instanceof TooLargeError) {
|
|
820
|
+
// over the input cap: split the document and send the parts
|
|
821
|
+
} else if (err instanceof CloudError) {
|
|
822
|
+
// anything else; err.status says what
|
|
823
|
+
} else {
|
|
824
|
+
throw err;
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
```
|
|
828
|
+
|
|
829
|
+
| Error | Status | When | Retried by the SDK |
|
|
830
|
+
|---|---|---|---|
|
|
831
|
+
| `NotConfiguredError` | — | no API key configured, or no global `fetch` | no |
|
|
832
|
+
| `CloudError` | 401 | the key is wrong or revoked | no |
|
|
833
|
+
| `CloudError` | 403 | the service refused the call. Account calls are accepted with an API key while the service does not require a browser `Origin`; if that changes they answer 403 | no |
|
|
834
|
+
| `CloudError` | 403 | `Batches.create` on a workspace without retention: `detail.code === "retention_required"` (see [Batches](#batches)) | no |
|
|
835
|
+
| `NotFoundError` | 404 | no such job, batch or key — or another account's; also every batch call, `list()` included, while batches are not enabled on the service (`batches_not_enabled`) | no |
|
|
836
|
+
| `CloudError` | 409 | `revokeKey` on a key already revoked | no |
|
|
837
|
+
| `ResultExpiredError` | 410 | the job's result is past retention | no |
|
|
838
|
+
| `TooLargeError` | 413 | the document is over the input cap; also a rejected job | no |
|
|
839
|
+
| `QuotaExceededError` | 429 | the daily document quota is used up | **no**: it will not clear before the day does |
|
|
840
|
+
| `RateLimitedError` | 429 | the edge's rate limit, still hit after every retry | yes, `maxRetries` times with backoff |
|
|
841
|
+
| `CloudError` | 5xx, `undefined` | the service or the network failed | yes for reads and `redactAsync`; never for `revokeKey` |
|
|
842
|
+
|
|
843
|
+
`NotFoundError`, `ResultExpiredError` and `RateLimitedError` are exported from
|
|
844
|
+
`euredact` alongside the others.
|
|
845
|
+
|
|
846
|
+
**Availability:** like the rest of the cloud tier, these calls are in private
|
|
847
|
+
alpha, and the response shapes may still change between releases. The batch
|
|
848
|
+
calls, `list()` included, answer `NotFoundError` until batches are enabled on
|
|
849
|
+
the service (see [Batches](#batches)).
|
|
850
|
+
|
|
851
|
+
### Batches
|
|
852
|
+
|
|
853
|
+
> **Availability:** private alpha, like the rest of the cloud tier. Batches
|
|
854
|
+
> need **retention** on the workspace (see below); while batches are not
|
|
855
|
+
> enabled on the service, every batch call answers `NotFoundError`
|
|
856
|
+
> (`batches_not_enabled`).
|
|
857
|
+
|
|
858
|
+
A batch hands up to 5,000 documents to the cloud tier and collects the answers
|
|
859
|
+
within 24 hours, at a lower price and without your plan's concurrency limit.
|
|
860
|
+
It keeps the same promise as `mode: "cloud"`: each document is masked **on
|
|
861
|
+
your machine**, and only the masked text is uploaded. What is needed to map the
|
|
862
|
+
answers back stays in a **local batch file**, `~/.euredact/batches/<batchId>.json`
|
|
863
|
+
(directory `0700`, file `0600`): the originals, their local detections and a
|
|
864
|
+
SHA-256 of the masked text sent. The file format is shared with the Python SDK,
|
|
865
|
+
so a batch created by one can be resolved by the other.
|
|
866
|
+
|
|
867
|
+
```ts
|
|
868
|
+
import { configure, Batches } from "euredact";
|
|
869
|
+
|
|
870
|
+
configure({ apiKey: "erk_..." });
|
|
871
|
+
const batches = new Batches();
|
|
872
|
+
const batch = await batches.create([
|
|
873
|
+
{ customId: "invoice-2291", text: invoiceText, countries: ["BE"] },
|
|
874
|
+
{ customId: "memo-17", text: memoText, countries: ["NL"] },
|
|
875
|
+
]);
|
|
876
|
+
|
|
877
|
+
// Later, possibly in another process:
|
|
878
|
+
const outcome = await batches.results(batch.id);
|
|
879
|
+
if (outcome.status === "resolved") {
|
|
880
|
+
for (const [customId, doc] of Object.entries(outcome.documents)) {
|
|
881
|
+
console.log(customId, doc.ok ? doc.result!.redactedText : `${doc.error}: ${doc.message}`);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
```
|
|
885
|
+
|
|
886
|
+
**Batches need retention.** The service holds each document until it is
|
|
887
|
+
processed and each result until you fetch it, so a workspace that erases
|
|
888
|
+
documents on delivery (`retentionMode === "none"`, or 0 hours of retention)
|
|
889
|
+
cannot create one. `create()` then throws `CloudError` with status 403 and
|
|
890
|
+
`detail.code === "retention_required"`, before anything is uploaded or written
|
|
891
|
+
locally. The workspace owner turns retention on in the console, on the
|
|
892
|
+
**Account** tab. To check first:
|
|
893
|
+
|
|
894
|
+
```ts
|
|
895
|
+
import { Account, CloudError } from "euredact";
|
|
896
|
+
|
|
897
|
+
if (!(await new Account().summary()).batchesAvailable) {
|
|
898
|
+
// no retention on this workspace: send the documents with mode: "cloud"
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
try {
|
|
902
|
+
await batches.create(documents);
|
|
903
|
+
} catch (err) {
|
|
904
|
+
if (err instanceof CloudError && err.status === 403 && err.detail?.code === "retention_required") {
|
|
905
|
+
// ask the workspace owner to turn retention on
|
|
906
|
+
}
|
|
907
|
+
throw err;
|
|
908
|
+
}
|
|
909
|
+
```
|
|
910
|
+
|
|
911
|
+
- **`results()`** returns `not_ended` while the batch runs and changes nothing.
|
|
912
|
+
Once it has ended it maps every document, wipes the originals and leaves a
|
|
913
|
+
text-free receipt; a second call returns `already_resolved`.
|
|
914
|
+
- **Integrity.** Each masked text is rebuilt from the file and must match the
|
|
915
|
+
hash the gateway reports before a span is placed. An edited file, a missing
|
|
916
|
+
entry or no file is an error for that document (`local_mismatch`,
|
|
917
|
+
`missing_local_entry`), never the masked text passed off as a result.
|
|
918
|
+
- **`pending()`**, **`retrieve(id)`**, **`cancel(id)`** (the file stays until
|
|
919
|
+
the finished documents are mapped) and **`purge(id)`**. Receipts and
|
|
920
|
+
unmapped files are cleaned up once the gateway's results expire.
|
|
921
|
+
- **In a browser** there is no filesystem: pass `store`, any object with async
|
|
922
|
+
`read`, `write`, `delete` and `list`. In Node, `batchDir` moves the
|
|
923
|
+
directory. `cipher` (`encrypt`/`decrypt`) encrypts the file at rest with a
|
|
924
|
+
key you hold.
|
|
925
|
+
- **Limits** checked before anything is masked or uploaded: 5,000 documents,
|
|
926
|
+
unique `customId`s of 1-64 characters from `A-Z a-z 0-9 _ . : -` (the
|
|
927
|
+
gateway's rule, exported as `CUSTOM_ID`), one country each, a 128 MB body. The 5,000-token limit
|
|
928
|
+
per document is **not** checked by the SDK: it is counted by the model's
|
|
929
|
+
tokenizer on the gateway. A document over it comes back as `too_long`, whose
|
|
930
|
+
message says so; split it and submit the parts.
|
|
931
|
+
|
|
593
932
|
## `NAME` is now `PERSON_NAME`
|
|
594
933
|
|
|
595
934
|
The canonical type name is `PERSON_NAME`; `NAME` is a legacy alias, exactly as
|
|
@@ -802,14 +1141,23 @@ BIC is the only bank identifier here with **no check digit**, and characters
|
|
|
802
1141
|
| Stage | Condition | Result |
|
|
803
1142
|
|---|---|---|
|
|
804
1143
|
| Gate 0 | the token also occurs as an ordinary lowercase word in the same document | never emitted |
|
|
805
|
-
| Tier 1 | registry hit on the BIC6 institution+country prefix | emitted |
|
|
1144
|
+
| Tier 1 | registry hit on the BIC6 institution+country prefix (your registry, or the bundled seed of major banks) | emitted |
|
|
806
1145
|
| Gate 2 | heading / shouted-word shape | never emitted |
|
|
807
|
-
| Tier 2 | `BIC`/`SWIFT` keyword, an IBAN, or a bank block in the enclosing line, record or paragraph | emitted |
|
|
1146
|
+
| Tier 2 | `BIC`/`SWIFT` keyword, an IBAN, or a bank block in the enclosing line, record or paragraph | emitted — but a **letters-only** code (`BETALING`, `HOFFMANN`), unless the bundled GLEIF mapping knows the institution, also needs a `BIC`/`SWIFT` label touching it |
|
|
808
1147
|
| — | none of the above | never emitted |
|
|
809
1148
|
|
|
810
|
-
The package bundles **
|
|
811
|
-
|
|
812
|
-
|
|
1149
|
+
The package bundles about **10,400 BIC6 prefixes** for the 31 supported countries,
|
|
1150
|
+
derived from the [GLEIF BIC-to-LEI mapping](https://www.gleif.org/en/lei-data/lei-mapping/download-bic-to-lei-relationship-files),
|
|
1151
|
+
which SWIFT develops and licenses for redistribution under the BIC/LEI Mapping
|
|
1152
|
+
Table License Agreement; its required notice is in `NOTICE`. They tell a real
|
|
1153
|
+
bank from an ordinary word next to an IBAN, but do not by themselves license a
|
|
1154
|
+
code in bare prose: with that many real prefixes, some words begin with one
|
|
1155
|
+
(`DERNIERS` → `DERN`+`IE`, an Irish institution). `scripts/refresh_bic_registry.py`
|
|
1156
|
+
regenerates the list; GLEIF publishes the mapping monthly.
|
|
1157
|
+
|
|
1158
|
+
The complete SWIFTRef BIC Directory is a commercial product and is not
|
|
1159
|
+
bundled. Deployments holding it install it at startup, where it works like the
|
|
1160
|
+
seed — a hit is emitted without context:
|
|
813
1161
|
|
|
814
1162
|
```ts
|
|
815
1163
|
import { setBicRegistry } from "euredact";
|
|
@@ -819,10 +1167,10 @@ setBicRegistry(bic => myDirectory.has(bic)); // membership callable
|
|
|
819
1167
|
setBicRegistry(null); // remove
|
|
820
1168
|
```
|
|
821
1169
|
|
|
822
|
-
The registry is an **accept** signal
|
|
823
|
-
|
|
824
|
-
present
|
|
825
|
-
|
|
1170
|
+
The registry is mostly an **accept** signal: a code missing from it falls
|
|
1171
|
+
through to the context gate and is still detected when banking context is
|
|
1172
|
+
present — except a letters-only code, the only shape an ordinary word can take,
|
|
1173
|
+
which then also needs a `BIC`/`SWIFT` label touching it (rules-engine#57).
|
|
826
1174
|
|
|
827
1175
|
## `IBAN` is now `BANK_ACCOUNT`
|
|
828
1176
|
|
|
@@ -996,8 +1344,8 @@ and two tokenized documents never reveal that they share a value. That is the
|
|
|
996
1344
|
opposite retention model from `referentialIntegrity`, which is why the two
|
|
997
1345
|
cannot be combined. `redactBatch` tokenizes each text on its own.
|
|
998
1346
|
|
|
999
|
-
A token is `TYPE_` plus four characters from `
|
|
1000
|
-
(no vowels
|
|
1347
|
+
A token is `TYPE_` plus four characters from `BCDFGHJKLMNPQRSTVWXZ23456789`
|
|
1348
|
+
(no vowels or `Y`, no `0`/`1`/`I`/`O`), about 615,000 suffixes. Tokens are kept clear of any token-shaped
|
|
1001
1349
|
string already in the document, so an LLM's reply to a tokenized prompt can
|
|
1002
1350
|
itself be redacted without `restore()` putting the wrong value back.
|
|
1003
1351
|
`restore()` replaces every occurrence, including a token an LLM glued to other
|
|
@@ -1175,11 +1523,12 @@ bundler, edge-runtime and Deno compatibility — would buy nothing.
|
|
|
1175
1523
|
|
|
1176
1524
|
| Package | |
|
|
1177
1525
|
|---|---:|
|
|
1178
|
-
| Tarball |
|
|
1179
|
-
| Unpacked |
|
|
1526
|
+
| Tarball | 232 kB |
|
|
1527
|
+
| Unpacked | 817 kB |
|
|
1180
1528
|
| Runtime dependencies | **0** |
|
|
1181
1529
|
|
|
1182
|
-
Measured with `npm pack --dry-run`
|
|
1530
|
+
Measured with `npm pack --dry-run` with the bundled BIC prefix list, which
|
|
1531
|
+
accounts for about 60 kB of the tarball (shipped in both the ESM and CJS builds). A stale duplicate of this section
|
|
1183
1532
|
previously quoted 0.02 ms and 86 KB; both were wrong, and the figures above are
|
|
1184
1533
|
the measured ones.
|
|
1185
1534
|
|
|
@@ -1189,6 +1538,21 @@ the measured ones.
|
|
|
1189
1538
|
const { redact } = require("euredact");
|
|
1190
1539
|
```
|
|
1191
1540
|
|
|
1541
|
+
## Browsers
|
|
1542
|
+
|
|
1543
|
+
The package bundles for a page as published. Its only platform dependency is a
|
|
1544
|
+
SHA-256 digest (the result-cache key) and a random index (token suffixes); on
|
|
1545
|
+
Node both come from `node:crypto`, and a bundler targeting the browser (esbuild,
|
|
1546
|
+
webpack, Rollup, Vite) follows the `browser` field in `package.json` to a build
|
|
1547
|
+
that uses `crypto.getRandomValues` and a synchronous SHA-256 of its own, checked
|
|
1548
|
+
against `node:crypto` byte for byte. `redact` runs entirely in the page, and so
|
|
1549
|
+
does the local half of `redactAsync({ mode: "cloud" })`: only the masked text
|
|
1550
|
+
leaves it.
|
|
1551
|
+
|
|
1552
|
+
```js
|
|
1553
|
+
import { redact } from "euredact"; // esbuild --platform=browser, webpack, Vite
|
|
1554
|
+
```
|
|
1555
|
+
|
|
1192
1556
|
## License
|
|
1193
1557
|
|
|
1194
1558
|
Apache-2.0
|
package/dist/cjs/cache.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ResultCache = void 0;
|
|
4
4
|
exports.resultChars = resultChars;
|
|
5
|
-
const
|
|
5
|
+
const platform_js_1 = require("./platform.js");
|
|
6
6
|
// Default ceiling on retained characters, not entries. A count-only cap of
|
|
7
7
|
// 1024 entries says nothing about memory: 1024 cached 1 MB documents is a
|
|
8
8
|
// gigabyte of PII-bearing text held live. ~16M characters is a few tens of MB.
|
|
@@ -37,13 +37,7 @@ class ResultCache {
|
|
|
37
37
|
// deployment an attacker who can guess a victim's document could submit a
|
|
38
38
|
// same-length colliding one and be served the victim's cached result,
|
|
39
39
|
// detections and raw matched PII included.
|
|
40
|
-
return (0,
|
|
41
|
-
.update(text)
|
|
42
|
-
.update("|")
|
|
43
|
-
.update(sorted.join("|"))
|
|
44
|
-
.update("|")
|
|
45
|
-
.update(mode)
|
|
46
|
-
.digest("hex");
|
|
40
|
+
return (0, platform_js_1.sha256Hex)([text, "|", sorted.join("|"), "|", mode]);
|
|
47
41
|
}
|
|
48
42
|
get(key) {
|
|
49
43
|
if (!this.enabled)
|