euredact 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -0
- package/README.md +658 -45
- package/dist/cjs/cache.js +2 -8
- package/dist/cjs/cloud/account.js +180 -0
- package/dist/cjs/cloud/batches.js +469 -0
- package/dist/cjs/cloud/client.js +131 -18
- package/dist/cjs/cloud/config.js +28 -2
- package/dist/cjs/cloud/errors.js +39 -2
- package/dist/cjs/cloud/index.js +16 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/platform.browser.js +98 -0
- package/dist/cjs/platform.js +140 -0
- package/dist/cjs/rules/bicGleif.js +57 -0
- package/dist/cjs/rules/bicRegistry.js +37 -11
- package/dist/cjs/rules/countries/index.js +138 -48
- package/dist/cjs/rules/cues.js +29 -9
- package/dist/cjs/rules/engine.js +14 -6
- package/dist/cjs/rules/suppressors.js +418 -23
- package/dist/cjs/rules/validators.js +103 -0
- package/dist/cjs/sdk.js +150 -44
- package/dist/cjs/types.js +4 -1
- package/dist/esm/cache.js +2 -8
- package/dist/esm/cloud/account.d.ts +155 -0
- package/dist/esm/cloud/account.js +175 -0
- package/dist/esm/cloud/batches.d.ts +166 -0
- package/dist/esm/cloud/batches.js +464 -0
- package/dist/esm/cloud/client.d.ts +90 -5
- package/dist/esm/cloud/client.js +126 -26
- package/dist/esm/cloud/config.d.ts +6 -0
- package/dist/esm/cloud/config.js +27 -2
- package/dist/esm/cloud/errors.d.ts +26 -1
- package/dist/esm/cloud/errors.js +35 -1
- package/dist/esm/cloud/index.d.ts +4 -2
- package/dist/esm/cloud/index.js +4 -2
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/platform.browser.d.ts +27 -0
- package/dist/esm/platform.browser.js +92 -0
- package/dist/esm/platform.d.ts +20 -0
- package/dist/esm/platform.js +101 -0
- package/dist/esm/rules/bicGleif.d.ts +4 -0
- package/dist/esm/rules/bicGleif.js +53 -0
- package/dist/esm/rules/bicRegistry.d.ts +25 -11
- package/dist/esm/rules/bicRegistry.js +36 -11
- package/dist/esm/rules/countries/index.js +138 -48
- package/dist/esm/rules/cues.d.ts +1 -1
- package/dist/esm/rules/cues.js +29 -9
- package/dist/esm/rules/engine.js +14 -6
- package/dist/esm/rules/suppressors.d.ts +1 -0
- package/dist/esm/rules/suppressors.js +417 -22
- package/dist/esm/rules/validators.d.ts +14 -0
- package/dist/esm/rules/validators.js +100 -0
- package/dist/esm/sdk.d.ts +63 -19
- package/dist/esm/sdk.js +148 -44
- package/dist/esm/types.d.ts +58 -2
- package/dist/esm/types.js +4 -1
- package/package.json +12 -4
package/dist/esm/cloud/client.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* [CLOUD EXTENSION] The cloud client.
|
|
3
3
|
*
|
|
4
|
-
* Speaks to the euRedact inference service: a
|
|
5
|
-
*
|
|
6
|
-
* service returns both the redacted document and located spans, so this
|
|
4
|
+
* Speaks to the euRedact inference service: a fine-tuned model asked only
|
|
5
|
+
* *what did the rules miss?* The service returns located spans, so this
|
|
7
6
|
* module's job is transport and translation, not detection.
|
|
8
7
|
*
|
|
8
|
+
* It sends the text it is given, as given. `redactAsync(text, { mode: "cloud" })`
|
|
9
|
+
* is what runs the rules engine first and hands this client only the masked
|
|
10
|
+
* text; call the client directly and whatever you pass is what leaves the
|
|
11
|
+
* machine. The spans that come back index the text that was sent.
|
|
12
|
+
*
|
|
9
13
|
* Three things it hides from the caller:
|
|
10
14
|
*
|
|
11
15
|
* - **Whether the document finished inside the service's sync window.** A short
|
|
@@ -20,12 +24,14 @@
|
|
|
20
24
|
* Zero dependencies, like the rest of the package: this uses the platform's
|
|
21
25
|
* own `fetch`.
|
|
22
26
|
*/
|
|
23
|
-
import { getConfig, } from "./config.js";
|
|
24
|
-
import { CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, } from "./errors.js";
|
|
27
|
+
import { getConfig, requireSecureBaseUrl, } from "./config.js";
|
|
28
|
+
import { CloudError, NotConfiguredError, NotFoundError, QuotaExceededError, RateLimitedError, ResultExpiredError, TooLargeError, } from "./errors.js";
|
|
25
29
|
import { canonicalType, DetectionSource, EntityType, } from "../types.js";
|
|
26
|
-
|
|
30
|
+
/** @internal Shared with cloud/account.ts. */
|
|
31
|
+
export const RETRY_STATUS = new Set([429, 500, 502, 503, 504]);
|
|
27
32
|
const MAX_BACKOFF_MS = 30_000;
|
|
28
|
-
|
|
33
|
+
/** @internal */
|
|
34
|
+
export function requireFetch(supplied) {
|
|
29
35
|
const impl = supplied ?? globalThis.fetch;
|
|
30
36
|
if (typeof impl !== "function") {
|
|
31
37
|
throw new NotConfiguredError("the cloud tier needs a global fetch. Node 18+ provides one; on Node 16 " +
|
|
@@ -33,10 +39,12 @@ function requireFetch(supplied) {
|
|
|
33
39
|
}
|
|
34
40
|
return impl;
|
|
35
41
|
}
|
|
36
|
-
|
|
42
|
+
/** @internal */
|
|
43
|
+
export function sleep(ms) {
|
|
37
44
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
38
45
|
}
|
|
39
|
-
|
|
46
|
+
/** @internal */
|
|
47
|
+
export function retryAfterMs(headers) {
|
|
40
48
|
const raw = headers?.get("Retry-After");
|
|
41
49
|
if (!raw)
|
|
42
50
|
return null;
|
|
@@ -48,12 +56,14 @@ function retryAfterMs(headers) {
|
|
|
48
56
|
* Full jitter. A synchronised retry storm from many clients is how a
|
|
49
57
|
* recovering service is knocked back over.
|
|
50
58
|
*/
|
|
51
|
-
|
|
59
|
+
/** @internal */
|
|
60
|
+
export function backoffMs(attempt, retryAfter) {
|
|
52
61
|
if (retryAfter !== null)
|
|
53
62
|
return Math.min(retryAfter, MAX_BACKOFF_MS);
|
|
54
63
|
return Math.min(MAX_BACKOFF_MS, Math.random() * 2 ** attempt * 1000);
|
|
55
64
|
}
|
|
56
|
-
|
|
65
|
+
/** @internal */
|
|
66
|
+
export function uuid() {
|
|
57
67
|
const c = globalThis.crypto;
|
|
58
68
|
if (c && typeof c.randomUUID === "function")
|
|
59
69
|
return c.randomUUID();
|
|
@@ -80,7 +90,8 @@ function toEntityType(raw) {
|
|
|
80
90
|
* Returns the identity when the text has none, which is the common case and
|
|
81
91
|
* costs one regex test.
|
|
82
92
|
*/
|
|
83
|
-
|
|
93
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
94
|
+
export function codePointOffsets(text) {
|
|
84
95
|
if (!/[\uD800-\uDBFF]/.test(text))
|
|
85
96
|
return cp => cp;
|
|
86
97
|
const units = [];
|
|
@@ -92,7 +103,98 @@ function codePointOffsets(text) {
|
|
|
92
103
|
units.push(i);
|
|
93
104
|
return cp => units[Math.min(cp, units.length - 1)];
|
|
94
105
|
}
|
|
95
|
-
|
|
106
|
+
/** @internal */
|
|
107
|
+
export function isInteger(value) {
|
|
108
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* The `usage` block, or undefined when absent or malformed. Shared rules with
|
|
112
|
+
* the Python SDK (`conformance/cloud_usage.json`): `tokens` and `credits` are
|
|
113
|
+
* integers, `billing_rate` a number, `factors` a list whose entries carry a
|
|
114
|
+
* string `code`. Anything else there drops the whole block rather than
|
|
115
|
+
* reporting a cost nobody can trust; unknown keys are ignored.
|
|
116
|
+
* @internal Exported for the conformance test.
|
|
117
|
+
*/
|
|
118
|
+
export function toUsage(raw) {
|
|
119
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw))
|
|
120
|
+
return undefined;
|
|
121
|
+
const u = raw;
|
|
122
|
+
if (!isInteger(u.tokens) || !isInteger(u.credits) || typeof u.billing_rate !== "number"
|
|
123
|
+
|| !Array.isArray(u.factors))
|
|
124
|
+
return undefined;
|
|
125
|
+
const factors = [];
|
|
126
|
+
for (const f of u.factors) {
|
|
127
|
+
if (f === null || typeof f !== "object" || Array.isArray(f))
|
|
128
|
+
return undefined;
|
|
129
|
+
const r = f;
|
|
130
|
+
if (typeof r.code !== "string")
|
|
131
|
+
return undefined;
|
|
132
|
+
const factor = { code: r.code, detail: typeof r.detail === "string" ? r.detail : "" };
|
|
133
|
+
if (Array.isArray(r.types))
|
|
134
|
+
factor.types = r.types.filter((t) => typeof t === "string");
|
|
135
|
+
factors.push(factor);
|
|
136
|
+
}
|
|
137
|
+
return { tokens: u.tokens, billingRate: u.billing_rate, credits: u.credits, factors };
|
|
138
|
+
}
|
|
139
|
+
function str(value) {
|
|
140
|
+
return typeof value === "string" ? value : null;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The service's account of a request beside its spans. Shared rules with the
|
|
144
|
+
* Python SDK (`conformance/cloud_result.json`): a field of the wrong type reads
|
|
145
|
+
* as absent, and an unlocated entry without string `text` is skipped.
|
|
146
|
+
* @internal
|
|
147
|
+
*/
|
|
148
|
+
export function toCloudInfo(payload) {
|
|
149
|
+
const unlocated = [];
|
|
150
|
+
for (const item of Array.isArray(payload.unlocated) ? payload.unlocated : []) {
|
|
151
|
+
if (item && typeof item === "object" && typeof item.text === "string") {
|
|
152
|
+
const t = item.type;
|
|
153
|
+
unlocated.push({
|
|
154
|
+
text: item.text,
|
|
155
|
+
entityType: toEntityType(typeof t === "string" ? t : "OTHER"),
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const usage = toUsage(payload.usage);
|
|
160
|
+
return {
|
|
161
|
+
jobId: str(payload.job_id),
|
|
162
|
+
modelVersion: str(payload.model_version),
|
|
163
|
+
...(usage ? { usage } : {}),
|
|
164
|
+
unlocated,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* A 429 from the gateway's daily quota rather than the edge's rate limit: the
|
|
169
|
+
* gateway answers in JSON with `detail.used` and `detail.limit`; nginx in HTML.
|
|
170
|
+
* @internal
|
|
171
|
+
*/
|
|
172
|
+
export function isQuota(payload) {
|
|
173
|
+
const detail = payload.detail;
|
|
174
|
+
return !!detail && typeof detail === "object" && "limit" in detail && "used" in detail;
|
|
175
|
+
}
|
|
176
|
+
/** The error for a non-success answer. @internal Shared with batches and account. */
|
|
177
|
+
export function cloudErrorFor(status, payload) {
|
|
178
|
+
const message = typeof payload.error === "string" ? payload.error : `HTTP ${status}`;
|
|
179
|
+
const detail = (payload.detail && typeof payload.detail === "object"
|
|
180
|
+
? payload.detail : {});
|
|
181
|
+
if (status === 401)
|
|
182
|
+
return new CloudError(`authentication failed: ${message}`, status);
|
|
183
|
+
if (status === 404)
|
|
184
|
+
return new NotFoundError(message, status, detail);
|
|
185
|
+
if (status === 410)
|
|
186
|
+
return new ResultExpiredError(message, status, detail);
|
|
187
|
+
if (status === 413)
|
|
188
|
+
return new TooLargeError(message, status, detail);
|
|
189
|
+
if (status === 429) {
|
|
190
|
+
return isQuota(payload)
|
|
191
|
+
? new QuotaExceededError(message, status, detail)
|
|
192
|
+
: new RateLimitedError("rate limited by the service (HTTP 429); try again shortly", status, detail);
|
|
193
|
+
}
|
|
194
|
+
return new CloudError(message, status, detail);
|
|
195
|
+
}
|
|
196
|
+
/** @internal Shared with cloud/batches.ts. */
|
|
197
|
+
export function toResult(payload, text) {
|
|
96
198
|
const offset = codePointOffsets(text);
|
|
97
199
|
const detections = (payload.entities ?? []).map(span => ({
|
|
98
200
|
entityType: toEntityType(span.type),
|
|
@@ -101,7 +203,9 @@ function toResult(payload, text) {
|
|
|
101
203
|
text: span.text,
|
|
102
204
|
source: span.source === "model" ? DetectionSource.CLOUD : DetectionSource.RULES,
|
|
103
205
|
country: null,
|
|
104
|
-
|
|
206
|
+
// A model span carries no evidence of its own: the service's confidence
|
|
207
|
+
// when it sends one, never an invented "high" (rules-engine#108).
|
|
208
|
+
confidence: span.source === "model" ? (span.confidence ?? null) : (span.confidence ?? "high"),
|
|
105
209
|
}));
|
|
106
210
|
detections.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
107
211
|
return {
|
|
@@ -114,6 +218,7 @@ function toResult(payload, text) {
|
|
|
114
218
|
detectionMode: "declared",
|
|
115
219
|
tokens: {},
|
|
116
220
|
exempted: [],
|
|
221
|
+
cloud: toCloudInfo(payload),
|
|
117
222
|
};
|
|
118
223
|
}
|
|
119
224
|
export class CloudClient {
|
|
@@ -122,6 +227,8 @@ export class CloudClient {
|
|
|
122
227
|
const resolved = config ?? getConfig();
|
|
123
228
|
if (resolved === null)
|
|
124
229
|
throw new NotConfiguredError();
|
|
230
|
+
// A config built by hand bypasses configure(), so check it here too.
|
|
231
|
+
requireSecureBaseUrl(resolved.baseUrl);
|
|
125
232
|
this.config = resolved;
|
|
126
233
|
}
|
|
127
234
|
headers(idempotencyKey) {
|
|
@@ -135,15 +242,7 @@ export class CloudClient {
|
|
|
135
242
|
};
|
|
136
243
|
}
|
|
137
244
|
raiseFor(status, payload) {
|
|
138
|
-
|
|
139
|
-
const detail = payload.detail ?? {};
|
|
140
|
-
if (status === 401)
|
|
141
|
-
throw new CloudError(`authentication failed: ${message}`, status);
|
|
142
|
-
if (status === 413)
|
|
143
|
-
throw new TooLargeError(message, status, detail);
|
|
144
|
-
if (status === 429)
|
|
145
|
-
throw new QuotaExceededError(message, status, detail);
|
|
146
|
-
throw new CloudError(message, status, detail);
|
|
245
|
+
throw cloudErrorFor(status, payload);
|
|
147
246
|
}
|
|
148
247
|
async redact(text, options) {
|
|
149
248
|
const doFetch = requireFetch(options.fetchImpl);
|
|
@@ -153,7 +252,6 @@ export class CloudClient {
|
|
|
153
252
|
country: options.country.toUpperCase(),
|
|
154
253
|
language: options.language ?? "",
|
|
155
254
|
priority: options.priority ?? "interactive",
|
|
156
|
-
rules_only: options.rulesOnly ?? false,
|
|
157
255
|
});
|
|
158
256
|
let attempt = 0;
|
|
159
257
|
for (;;) {
|
|
@@ -183,7 +281,8 @@ export class CloudClient {
|
|
|
183
281
|
}
|
|
184
282
|
if (response.status === 200)
|
|
185
283
|
return toResult(payload, text);
|
|
186
|
-
if (!RETRY_STATUS.has(response.status) || attempt >= this.config.maxRetries
|
|
284
|
+
if (!RETRY_STATUS.has(response.status) || attempt >= this.config.maxRetries
|
|
285
|
+
|| (response.status === 429 && isQuota(payload))) {
|
|
187
286
|
this.raiseFor(response.status, payload);
|
|
188
287
|
}
|
|
189
288
|
}
|
|
@@ -228,7 +327,8 @@ export class CloudClient {
|
|
|
228
327
|
}
|
|
229
328
|
}
|
|
230
329
|
}
|
|
231
|
-
|
|
330
|
+
/** @internal */
|
|
331
|
+
export async function readJson(response) {
|
|
232
332
|
try {
|
|
233
333
|
const data = await response.json();
|
|
234
334
|
return data && typeof data === "object" ? data : {};
|
|
@@ -30,6 +30,12 @@ export interface ConfigureOptions {
|
|
|
30
30
|
maxRetries?: number;
|
|
31
31
|
headers?: Record<string, string>;
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Refuse a cloud address that would not use TLS. The API key and the text sent
|
|
35
|
+
* must travel encrypted; plain `http` is allowed only to a loopback host, for a
|
|
36
|
+
* local gateway or a TLS-terminating proxy on the same machine (rules-engine#86).
|
|
37
|
+
*/
|
|
38
|
+
export declare function requireSecureBaseUrl(baseUrl: string): void;
|
|
33
39
|
/**
|
|
34
40
|
* Configure the cloud tier.
|
|
35
41
|
*
|
package/dist/esm/cloud/config.js
CHANGED
|
@@ -13,6 +13,29 @@ export const DEFAULT_TIMEOUT_MS = 90_000;
|
|
|
13
13
|
*/
|
|
14
14
|
export const DEFAULT_POLL_TIMEOUT_MS = 900_000;
|
|
15
15
|
export const DEFAULT_MAX_RETRIES = 3;
|
|
16
|
+
/** Hosts a plain-HTTP base URL may name: a gateway or TLS proxy on this machine. */
|
|
17
|
+
const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]", "::1"]);
|
|
18
|
+
/**
|
|
19
|
+
* Refuse a cloud address that would not use TLS. The API key and the text sent
|
|
20
|
+
* must travel encrypted; plain `http` is allowed only to a loopback host, for a
|
|
21
|
+
* local gateway or a TLS-terminating proxy on the same machine (rules-engine#86).
|
|
22
|
+
*/
|
|
23
|
+
export function requireSecureBaseUrl(baseUrl) {
|
|
24
|
+
// Parsed by hand rather than with `URL`: a runtime without that global would
|
|
25
|
+
// otherwise refuse every address, https included, with a misleading message.
|
|
26
|
+
const m = /^([a-z][a-z0-9+.-]*):\/\/(?:[^@/?#]*@)?(\[[^\]]*\]|[^:/?#]*)/i.exec(baseUrl);
|
|
27
|
+
if (m !== null) {
|
|
28
|
+
const scheme = m[1].toLowerCase();
|
|
29
|
+
const host = m[2].toLowerCase();
|
|
30
|
+
if (scheme === "https" && host)
|
|
31
|
+
return;
|
|
32
|
+
if (scheme === "http" && LOOPBACK_HOSTS.has(host))
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
throw new Error(`refusing cloud base URL ${JSON.stringify(baseUrl)}: it must start with https:// ` +
|
|
36
|
+
"(plain http:// is allowed only to localhost, 127.0.0.1 or ::1), because the " +
|
|
37
|
+
"API key and document text would otherwise travel unencrypted");
|
|
38
|
+
}
|
|
16
39
|
let config = null;
|
|
17
40
|
/** Read an env var where there is an environment to read. Browsers have none. */
|
|
18
41
|
function env(name) {
|
|
@@ -30,10 +53,12 @@ export function configure(options = {}) {
|
|
|
30
53
|
if (!apiKey) {
|
|
31
54
|
throw new Error("no API key: pass configure({ apiKey }) or set EUREDACT_API_KEY");
|
|
32
55
|
}
|
|
56
|
+
const baseUrl = (options.baseUrl ?? env("EUREDACT_BASE_URL") ?? DEFAULT_BASE_URL)
|
|
57
|
+
.replace(/\/+$/, "");
|
|
58
|
+
requireSecureBaseUrl(baseUrl);
|
|
33
59
|
config = {
|
|
34
60
|
apiKey,
|
|
35
|
-
baseUrl
|
|
36
|
-
.replace(/\/+$/, ""),
|
|
61
|
+
baseUrl,
|
|
37
62
|
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
38
63
|
pollTimeoutMs: options.pollTimeoutMs ?? DEFAULT_POLL_TIMEOUT_MS,
|
|
39
64
|
maxRetries: options.maxRetries ?? DEFAULT_MAX_RETRIES,
|
|
@@ -16,10 +16,35 @@ export declare class CloudError extends Error {
|
|
|
16
16
|
export declare class NotConfiguredError extends CloudError {
|
|
17
17
|
constructor(message?: string);
|
|
18
18
|
}
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* 429: the account's daily document quota is used up. Thrown at once, not
|
|
21
|
+
* after retries: the gateway's quota answer (JSON with `detail.used` and
|
|
22
|
+
* `detail.limit`) will not change before the day does (rules-engine#89).
|
|
23
|
+
*/
|
|
20
24
|
export declare class QuotaExceededError extends CloudError {
|
|
21
25
|
constructor(message: string, status?: number, detail?: Record<string, unknown>);
|
|
22
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* 429 from the edge's rate limit, still there after every retry. It clears
|
|
29
|
+
* within seconds, so it is retried with backoff first. A subclass of
|
|
30
|
+
* `QuotaExceededError`, which is what every 429 threw before the two were told apart.
|
|
31
|
+
*/
|
|
32
|
+
export declare class RateLimitedError extends QuotaExceededError {
|
|
33
|
+
constructor(message: string, status?: number, detail?: Record<string, unknown>);
|
|
34
|
+
}
|
|
35
|
+
/** 404: no such job, batch or key -- or another account's, which the service
|
|
36
|
+
* deliberately does not distinguish. */
|
|
37
|
+
export declare class NotFoundError extends CloudError {
|
|
38
|
+
constructor(message: string, status?: number, detail?: Record<string, unknown>);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* 410: the job succeeded, but its result is no longer retained -- swept by the
|
|
42
|
+
* account's retention period, or discarded on delivery under no-retention. An
|
|
43
|
+
* empty result here would read like a document with no personal data in it.
|
|
44
|
+
*/
|
|
45
|
+
export declare class ResultExpiredError extends CloudError {
|
|
46
|
+
constructor(message: string, status?: number, detail?: Record<string, unknown>);
|
|
47
|
+
}
|
|
23
48
|
/**
|
|
24
49
|
* 413. The document is over the service's input cap.
|
|
25
50
|
*
|
package/dist/esm/cloud/errors.js
CHANGED
|
@@ -26,13 +26,47 @@ export class NotConfiguredError extends CloudError {
|
|
|
26
26
|
this.name = "NotConfiguredError";
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* 429: the account's daily document quota is used up. Thrown at once, not
|
|
31
|
+
* after retries: the gateway's quota answer (JSON with `detail.used` and
|
|
32
|
+
* `detail.limit`) will not change before the day does (rules-engine#89).
|
|
33
|
+
*/
|
|
30
34
|
export class QuotaExceededError extends CloudError {
|
|
31
35
|
constructor(message, status, detail) {
|
|
32
36
|
super(message, status, detail);
|
|
33
37
|
this.name = "QuotaExceededError";
|
|
34
38
|
}
|
|
35
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* 429 from the edge's rate limit, still there after every retry. It clears
|
|
42
|
+
* within seconds, so it is retried with backoff first. A subclass of
|
|
43
|
+
* `QuotaExceededError`, which is what every 429 threw before the two were told apart.
|
|
44
|
+
*/
|
|
45
|
+
export class RateLimitedError extends QuotaExceededError {
|
|
46
|
+
constructor(message, status, detail) {
|
|
47
|
+
super(message, status, detail);
|
|
48
|
+
this.name = "RateLimitedError";
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** 404: no such job, batch or key -- or another account's, which the service
|
|
52
|
+
* deliberately does not distinguish. */
|
|
53
|
+
export class NotFoundError extends CloudError {
|
|
54
|
+
constructor(message, status, detail) {
|
|
55
|
+
super(message, status, detail);
|
|
56
|
+
this.name = "NotFoundError";
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* 410: the job succeeded, but its result is no longer retained -- swept by the
|
|
61
|
+
* account's retention period, or discarded on delivery under no-retention. An
|
|
62
|
+
* empty result here would read like a document with no personal data in it.
|
|
63
|
+
*/
|
|
64
|
+
export class ResultExpiredError extends CloudError {
|
|
65
|
+
constructor(message, status, detail) {
|
|
66
|
+
super(message, status, detail);
|
|
67
|
+
this.name = "ResultExpiredError";
|
|
68
|
+
}
|
|
69
|
+
}
|
|
36
70
|
/**
|
|
37
71
|
* 413. The document is over the service's input cap.
|
|
38
72
|
*
|
|
@@ -15,5 +15,7 @@
|
|
|
15
15
|
* ```
|
|
16
16
|
*/
|
|
17
17
|
export { CloudClient, type CloudRedactOptions } from "./client.js";
|
|
18
|
-
export {
|
|
19
|
-
export {
|
|
18
|
+
export { Account, Jobs, type AccountSummary, type ApiClientOptions, type ApiKey, type CreditEntry, type Credits, type Job, type KeyRevocation, type KeyUsage, type UsageDay, } from "./account.js";
|
|
19
|
+
export { Batches, BatchError, MAX_DOCUMENTS, MAX_CUSTOM_ID, CUSTOM_ID, MAX_BODY_BYTES, MAX_DOCUMENT_TOKENS, type Batch, type BatchDocument, type BatchResults, type BatchStore, type BatchesOptions, type Cipher, type DocumentOutcome, } from "./batches.js";
|
|
20
|
+
export { configure, getConfig, reset, requireSecureBaseUrl, DEFAULT_BASE_URL, type CloudConfig, type ConfigureOptions, } from "./config.js";
|
|
21
|
+
export { CloudError, NotConfiguredError, NotFoundError, QuotaExceededError, RateLimitedError, ResultExpiredError, TooLargeError, } from "./errors.js";
|
package/dist/esm/cloud/index.js
CHANGED
|
@@ -15,5 +15,7 @@
|
|
|
15
15
|
* ```
|
|
16
16
|
*/
|
|
17
17
|
export { CloudClient } from "./client.js";
|
|
18
|
-
export {
|
|
19
|
-
export {
|
|
18
|
+
export { Account, Jobs, } from "./account.js";
|
|
19
|
+
export { Batches, BatchError, MAX_DOCUMENTS, MAX_CUSTOM_ID, CUSTOM_ID, MAX_BODY_BYTES, MAX_DOCUMENT_TOKENS, } from "./batches.js";
|
|
20
|
+
export { configure, getConfig, reset, requireSecureBaseUrl, DEFAULT_BASE_URL, } from "./config.js";
|
|
21
|
+
export { CloudError, NotConfiguredError, NotFoundError, QuotaExceededError, RateLimitedError, ResultExpiredError, TooLargeError, } from "./errors.js";
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES, canonicalType } from "./types.js";
|
|
2
|
-
export type { Detection, Exemption, RedactResult, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
|
|
2
|
+
export type { CloudInfo, Detection, Exemption, RedactResult, Unlocated, Usage, UsageFactor, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
|
|
3
3
|
export { DocumentContext } from "./rules/context.js";
|
|
4
4
|
export { EuRedact, restore, type RedactOptions } from "./sdk.js";
|
|
5
5
|
export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
|
|
6
6
|
export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES, type BicRegistryProvider } from "./rules/bicRegistry.js";
|
|
7
|
-
export { configure, getConfig, CloudClient, CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, type CloudConfig, type ConfigureOptions, type CloudRedactOptions, } from "./cloud/index.js";
|
|
7
|
+
export { configure, getConfig, Account, CloudClient, CloudError, Batches, BatchError, CUSTOM_ID, Jobs, NotConfiguredError, NotFoundError, QuotaExceededError, RateLimitedError, ResultExpiredError, TooLargeError, type AccountSummary, type ApiKey, type CreditEntry, type Credits, type Job, type KeyRevocation, type KeyUsage, type UsageDay, type CloudConfig, type ConfigureOptions, type CloudRedactOptions, type Batch, type BatchDocument, type BatchResults, type BatchStore, type BatchesOptions, type Cipher, type DocumentOutcome, } from "./cloud/index.js";
|
|
8
8
|
import { type RedactOptions } from "./sdk.js";
|
|
9
9
|
import type { RedactResult } from "./types.js";
|
|
10
10
|
export declare function availableCountries(): string[];
|
package/dist/esm/index.js
CHANGED
|
@@ -3,7 +3,7 @@ export { DocumentContext } from "./rules/context.js";
|
|
|
3
3
|
export { EuRedact, restore } from "./sdk.js";
|
|
4
4
|
export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
|
|
5
5
|
export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES } from "./rules/bicRegistry.js";
|
|
6
|
-
export { configure, getConfig, CloudClient, CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, } from "./cloud/index.js";
|
|
6
|
+
export { configure, getConfig, Account, CloudClient, CloudError, Batches, BatchError, CUSTOM_ID, Jobs, NotConfiguredError, NotFoundError, QuotaExceededError, RateLimitedError, ResultExpiredError, TooLargeError, } from "./cloud/index.js";
|
|
7
7
|
import { EuRedact } from "./sdk.js";
|
|
8
8
|
import { COUNTRY_CONFIGS } from "./rules/countries/index.js";
|
|
9
9
|
let _instance = null;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `platform.ts` for browsers: no `node:` imports, so the package bundles for a
|
|
3
|
+
* page. Selected by the `browser` field in package.json (rules-engine#56).
|
|
4
|
+
*
|
|
5
|
+
* The cache key is computed synchronously and must stay collision-resistant
|
|
6
|
+
* (see `ResultCache.key`), and Web Crypto's `subtle.digest` is asynchronous, so
|
|
7
|
+
* SHA-256 is implemented here. A test checks it against `node:crypto` byte for
|
|
8
|
+
* byte, astral characters and lone surrogates included.
|
|
9
|
+
*/
|
|
10
|
+
/** Hex SHA-256 of the UTF-8 encoding of `parts`, joined with nothing between. */
|
|
11
|
+
export declare function sha256Hex(parts: string[]): string;
|
|
12
|
+
/**
|
|
13
|
+
* A uniformly random integer in `[0, n)`, from `crypto.getRandomValues`.
|
|
14
|
+
* Rejection sampling: a plain modulo would favour the low residues.
|
|
15
|
+
*/
|
|
16
|
+
export declare function randomIndex(n: number): number;
|
|
17
|
+
/** Mirrors `PlatformBatchStore` in platform.ts (rules-engine#84). */
|
|
18
|
+
export interface PlatformBatchStore {
|
|
19
|
+
read(batchId: string): Promise<Uint8Array | null>;
|
|
20
|
+
write(batchId: string, data: Uint8Array): Promise<void>;
|
|
21
|
+
delete(batchId: string): Promise<void>;
|
|
22
|
+
list(): Promise<string[]>;
|
|
23
|
+
}
|
|
24
|
+
/** A page has no filesystem: the caller passes `store` to `Batches`. */
|
|
25
|
+
export declare function defaultBatchStore(_directory?: string): PlatformBatchStore | null;
|
|
26
|
+
/** No synchronous gzip in a page; the batch is uploaded uncompressed. */
|
|
27
|
+
export declare function gzip(_data: Uint8Array): Uint8Array | null;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `platform.ts` for browsers: no `node:` imports, so the package bundles for a
|
|
3
|
+
* page. Selected by the `browser` field in package.json (rules-engine#56).
|
|
4
|
+
*
|
|
5
|
+
* The cache key is computed synchronously and must stay collision-resistant
|
|
6
|
+
* (see `ResultCache.key`), and Web Crypto's `subtle.digest` is asynchronous, so
|
|
7
|
+
* SHA-256 is implemented here. A test checks it against `node:crypto` byte for
|
|
8
|
+
* byte, astral characters and lone surrogates included.
|
|
9
|
+
*/
|
|
10
|
+
const K = new Uint32Array([
|
|
11
|
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
|
12
|
+
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
|
13
|
+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
|
14
|
+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
|
15
|
+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
|
16
|
+
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
17
|
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
|
18
|
+
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
|
|
19
|
+
]);
|
|
20
|
+
/** Hex SHA-256 of the UTF-8 encoding of `parts`, joined with nothing between. */
|
|
21
|
+
export function sha256Hex(parts) {
|
|
22
|
+
// TextEncoder replaces a lone surrogate with U+FFFD, as Node's hash.update does.
|
|
23
|
+
const bytes = new TextEncoder().encode(parts.join(""));
|
|
24
|
+
const bitLength = bytes.length * 8;
|
|
25
|
+
const padded = new Uint8Array(((bytes.length + 9 + 63) >> 6) << 6);
|
|
26
|
+
padded.set(bytes);
|
|
27
|
+
padded[bytes.length] = 0x80;
|
|
28
|
+
const view = new DataView(padded.buffer);
|
|
29
|
+
view.setUint32(padded.length - 8, Math.floor(bitLength / 0x100000000));
|
|
30
|
+
view.setUint32(padded.length - 4, bitLength >>> 0);
|
|
31
|
+
const h = new Uint32Array([
|
|
32
|
+
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
|
|
33
|
+
]);
|
|
34
|
+
const w = new Uint32Array(64);
|
|
35
|
+
for (let offset = 0; offset < padded.length; offset += 64) {
|
|
36
|
+
for (let i = 0; i < 16; i++)
|
|
37
|
+
w[i] = view.getUint32(offset + i * 4);
|
|
38
|
+
for (let i = 16; i < 64; i++) {
|
|
39
|
+
const a = w[i - 15], b = w[i - 2];
|
|
40
|
+
const s0 = ((a >>> 7) | (a << 25)) ^ ((a >>> 18) | (a << 14)) ^ (a >>> 3);
|
|
41
|
+
const s1 = ((b >>> 17) | (b << 15)) ^ ((b >>> 19) | (b << 13)) ^ (b >>> 10);
|
|
42
|
+
w[i] = (w[i - 16] + s0 + w[i - 7] + s1) >>> 0;
|
|
43
|
+
}
|
|
44
|
+
let [a, b, c, d, e, f, g, hh] = h;
|
|
45
|
+
for (let i = 0; i < 64; i++) {
|
|
46
|
+
const S1 = ((e >>> 6) | (e << 26)) ^ ((e >>> 11) | (e << 21)) ^ ((e >>> 25) | (e << 7));
|
|
47
|
+
const ch = (e & f) ^ (~e & g);
|
|
48
|
+
const t1 = (hh + S1 + ch + K[i] + w[i]) >>> 0;
|
|
49
|
+
const S0 = ((a >>> 2) | (a << 30)) ^ ((a >>> 13) | (a << 19)) ^ ((a >>> 22) | (a << 10));
|
|
50
|
+
const maj = (a & b) ^ (a & c) ^ (b & c);
|
|
51
|
+
const t2 = (S0 + maj) >>> 0;
|
|
52
|
+
hh = g;
|
|
53
|
+
g = f;
|
|
54
|
+
f = e;
|
|
55
|
+
e = (d + t1) >>> 0;
|
|
56
|
+
d = c;
|
|
57
|
+
c = b;
|
|
58
|
+
b = a;
|
|
59
|
+
a = (t1 + t2) >>> 0;
|
|
60
|
+
}
|
|
61
|
+
h[0] = (h[0] + a) >>> 0;
|
|
62
|
+
h[1] = (h[1] + b) >>> 0;
|
|
63
|
+
h[2] = (h[2] + c) >>> 0;
|
|
64
|
+
h[3] = (h[3] + d) >>> 0;
|
|
65
|
+
h[4] = (h[4] + e) >>> 0;
|
|
66
|
+
h[5] = (h[5] + f) >>> 0;
|
|
67
|
+
h[6] = (h[6] + g) >>> 0;
|
|
68
|
+
h[7] = (h[7] + hh) >>> 0;
|
|
69
|
+
}
|
|
70
|
+
return Array.from(h, x => x.toString(16).padStart(8, "0")).join("");
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* A uniformly random integer in `[0, n)`, from `crypto.getRandomValues`.
|
|
74
|
+
* Rejection sampling: a plain modulo would favour the low residues.
|
|
75
|
+
*/
|
|
76
|
+
export function randomIndex(n) {
|
|
77
|
+
const limit = Math.floor(0x100000000 / n) * n;
|
|
78
|
+
const buf = new Uint32Array(1);
|
|
79
|
+
for (;;) {
|
|
80
|
+
globalThis.crypto.getRandomValues(buf);
|
|
81
|
+
if (buf[0] < limit)
|
|
82
|
+
return buf[0] % n;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** A page has no filesystem: the caller passes `store` to `Batches`. */
|
|
86
|
+
export function defaultBatchStore(_directory) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
/** No synchronous gzip in a page; the batch is uploaded uncompressed. */
|
|
90
|
+
export function gzip(_data) {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Hex SHA-256 of the UTF-8 encoding of `parts`, joined with nothing between. */
|
|
2
|
+
export declare function sha256Hex(parts: string[]): string;
|
|
3
|
+
/** A uniformly random integer in `[0, n)`. */
|
|
4
|
+
export declare function randomIndex(n: number): number;
|
|
5
|
+
/** Where a local batch file lives (rules-engine#84). Mirrors `BatchStore` in
|
|
6
|
+
* cloud/batches.ts; declared here so this module imports nothing from there. */
|
|
7
|
+
export interface PlatformBatchStore {
|
|
8
|
+
read(batchId: string): Promise<Uint8Array | null>;
|
|
9
|
+
write(batchId: string, data: Uint8Array): Promise<void>;
|
|
10
|
+
delete(batchId: string): Promise<void>;
|
|
11
|
+
list(): Promise<string[]>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* `<dir>/<batchId>.json`, directory `0700`, files `0600`, the default local
|
|
15
|
+
* batch store. Writes go to a temporary file in the same directory, created
|
|
16
|
+
* `0600` before any byte is written, then renamed over the old one.
|
|
17
|
+
*/
|
|
18
|
+
export declare function defaultBatchStore(directory?: string): PlatformBatchStore | null;
|
|
19
|
+
/** Gzip, for the batch upload. */
|
|
20
|
+
export declare function gzip(data: Uint8Array): Uint8Array | null;
|