euredact 0.6.0 → 0.7.0

package/dist/esm/cloud/batches.js

@@ -0,0 +1,464 @@
+/**
+ * [CLOUD EXTENSION] Batches: cloud mode, with hours between masking and mapping
+ * (rules-engine#84; the gateway side is `docs/BATCHES.md` in euredact-inference).
+ *
+ * Cloud mode masks a document here, sends only the masked text and maps the
+ * service's answer back onto the original, all in one call. A batch returns up
+ * to 24 h later, so `create()` writes one **local batch file** per batch: each
+ * original, its local detections and the SHA-256 of the masked text that was
+ * sent. None of it is uploaded. `results()` maps the service's spans back from
+ * that file, checking the hash before placing a span, then wipes the originals
+ * and leaves a text-free receipt.
+ *
+ * The file format is shared with the Python SDK, offsets counted in code points,
+ * so a batch created by one can be resolved by the other.
+ *
+ * In Node the file is `~/.euredact/batches/<batchId>.json` (`0600` in a `0700`
+ * directory); `batchDir` moves it. A page has no filesystem, so there `store`
+ * is required: any object with async `read`, `write`, `delete` and `list`.
+ * `cipher` encrypts the file at rest with a key the caller holds.
+ */
+import { EuRedact, applyReplacements, maskForCloud, ontoOriginal } from "../sdk.js";
+import { defaultBatchStore, gzip, sha256Hex } from "../platform.js";
+import { cloudErrorFor, codePointOffsets, isInteger, toCloudInfo, toResult } from "./client.js";
+import { getConfig, requireSecureBaseUrl } from "./config.js";
+import { CloudError, NotConfiguredError } from "./errors.js";
+/** The gateway's limits (docs/BATCHES.md §2-§3), checked before upload. */
+export const MAX_DOCUMENTS = 5000;
+export const MAX_CUSTOM_ID = 64;
+/** The gateway's customId rule (euredact-inference gateway/batches.py), whole string. */
+export const CUSTOM_ID = /^[A-Za-z0-9_.:\-]{1,64}$/;
+export const MAX_BODY_BYTES = 128 * 1024 * 1024;
+/**
+ * Per document, counted by the model's tokenizer on the gateway. The SDK has no
+ * copy of it and no character count decides it, so it is **not** checked before
+ * upload: a document over it comes back as an error with code `too_long`.
+ */
+export const MAX_DOCUMENT_TOKENS = 5000;
+export const FILE_FORMAT = "euredact-batch/1";
+export class BatchError extends CloudError {
+    constructor(message) {
+        super(message);
+        this.name = "BatchError";
+    }
+}
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+function now() {
+    return new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+/** UTF-16 offset -> code point offset, for writing the shared file format. */
+function utf16ToCodePoints(text) {
+    if (!/[\uD800-\uDBFF]/.test(text))
+        return unit => unit;
+    const map = new Map();
+    let cp = 0;
+    let unit = 0;
+    for (const ch of text) {
+        map.set(unit, cp);
+        unit += ch.length;
+        cp += 1;
+    }
+    map.set(unit, cp);
+    return u => map.get(u) ?? u;
+}
+function toStored(d, cp) {
+    return {
+        type: String(d.entityType), start: cp(d.start), end: cp(d.end), text: d.text,
+        source: String(d.source), country: d.country ?? null, confidence: d.confidence ?? "high",
+        country_confidence: d.countryConfidence ?? 0, out_of_scope: d.outOfScope ?? false,
+    };
+}
+function fromStored(raw, unit) {
+    return {
+        entityType: raw.type, start: unit(raw.start), end: unit(raw.end),
+        text: raw.text, source: raw.source, country: raw.country,
+        confidence: raw.confidence, countryConfidence: raw.country_confidence,
+        outOfScope: raw.out_of_scope,
+    };
+}
+// ── the client ────────────────────────────────────────────────────────────
+export class Batches {
+    config;
+    store;
+    cipher;
+    fetchImpl;
+    sdk;
+    constructor(options = {}) {
+        const config = options.config ?? getConfig();
+        if (config === null)
+            throw new NotConfiguredError();
+        requireSecureBaseUrl(config.baseUrl);
+        this.config = config;
+        if (options.batchDir !== undefined && options.store !== undefined) {
+            throw new Error("pass batchDir or store, not both");
+        }
+        const store = options.store ?? defaultBatchStore(options.batchDir);
+        if (store === null) {
+            throw new Error("no filesystem here: pass a store ({ read, write, delete, list }) to Batches");
+        }
+        this.store = store;
+        this.cipher = options.cipher ?? null;
+        this.fetchImpl = options.fetchImpl;
+        this.sdk = options.sdk ?? null;
+    }
+    // -- plumbing ---------------------------------------------------------
+    engine() {
+        if (this.sdk === null)
+            this.sdk = new EuRedact();
+        return this.sdk;
+    }
+    async request(method, path, body, headers = {}) {
+        const impl = this.fetchImpl ?? globalThis.fetch;
+        if (!impl)
+            throw new CloudError("no fetch implementation available");
+        try {
+            return await impl(`${this.config.baseUrl}${path}`, {
+                method,
+                body: body,
+                headers: {
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    "Idempotency-Key": headers["Idempotency-Key"] ?? crypto.randomUUID(),
+                    ...this.config.headers,
+                    ...headers,
+                },
+            });
+        }
+        catch (err) {
+            throw new CloudError(`batch request failed: ${err.message}`);
+        }
+    }
+    async json(resp) {
+        try {
+            return (await resp.json());
+        }
+        catch {
+            return {};
+        }
+    }
+    raiseFor(status, payload) {
+        throw cloudErrorFor(status, payload);
+    }
+    async load(batchId) {
+        let data = await this.store.read(batchId);
+        if (data === null)
+            return null;
+        if (this.cipher)
+            data = await this.cipher.decrypt(data);
+        const state = JSON.parse(decoder.decode(data));
+        if (state.format !== FILE_FORMAT) {
+            throw new BatchError(`local batch file ${batchId}: unknown format ${JSON.stringify(state.format)}`);
+        }
+        return state;
+    }
+    async save(state) {
+        let data = encoder.encode(JSON.stringify(state));
+        if (this.cipher)
+            data = await this.cipher.encrypt(data);
+        await this.store.write(state.batch_id, data);
+    }
+    static batch(raw) {
+        const str = (k) => (typeof raw[k] === "string" ? raw[k] : null);
+        return {
+            id: str("id") ?? "", status: str("status") ?? "",
+            createdAt: str("created_at"), expiresAt: str("expires_at"), endedAt: str("ended_at"),
+            resultsExpireAt: str("results_expire_at"),
+            counts: { ...(raw.counts ?? {}) },
+            documents: isInteger(raw.documents) ? raw.documents : null,
+            tokens: Object.fromEntries(Object.entries(raw.tokens && typeof raw.tokens === "object" ? raw.tokens : {}).filter((e) => isInteger(e[1]))),
+            creditsCharged: isInteger(raw.credits_charged) ? raw.credits_charged : null,
+            billingRate: typeof raw.billing_rate === "number" ? raw.billing_rate : null,
+            raw,
+        };
+    }
+    // -- create -----------------------------------------------------------
+    /**
+     * Mask every document here, upload only the masked text, keep the rest in
+     * the local batch file. The document count, `customId`s, countries and body
+     * size are checked before anything is uploaded or written. The 5,000-token
+     * limit per document is **not**: it is counted by the model's tokenizer on
+     * the gateway, which the SDK does not have, so a document over it comes back
+     * from `results()` as an error with code `too_long` that says so.
+     */
+    async create(documents, options = {}) {
+        if (documents.length === 0)
+            throw new BatchError("a batch needs at least one document");
+        if (documents.length > MAX_DOCUMENTS) {
+            throw new BatchError(`${documents.length.toLocaleString("en-US")} documents; a batch holds at most ` +
+                `${MAX_DOCUMENTS.toLocaleString("en-US")}`);
+        }
+        // Every document is checked before any is masked: masking 5,000 documents
+        // and then learning that the gateway refuses the batch for one bad customId
+        // wastes the whole pass (rules-engine#88).
+        const seen = new Set();
+        const checked = [];
+        documents.forEach((doc, index) => {
+            const number = index + 1;
+            const customId = doc.customId;
+            if (typeof customId !== "string" || !CUSTOM_ID.test(customId)) {
+                throw new BatchError(`document ${number}: customId must be 1-${MAX_CUSTOM_ID} characters of A-Z a-z 0-9 _ . : -`);
+            }
+            if (seen.has(customId)) {
+                throw new BatchError(`document ${number}: duplicate customId ${JSON.stringify(customId)}`);
+            }
+            seen.add(customId);
+            if (typeof doc.text !== "string")
+                throw new BatchError(`document ${number}: text must be a string`);
+            const countries = (doc.countries ?? []).map(c => c.toUpperCase());
+            if (countries.length !== 1) {
+                throw new BatchError(`document ${number}: exactly one country is needed, as in cloud mode`);
+            }
+            checked.push({ customId, text: doc.text, countries, language: doc.language ?? "" });
+        });
+        const lines = [];
+        const entries = {};
+        const engine = this.engine();
+        for (const { customId, text, countries, language } of checked) {
+            // The same local pass as redactAsync(..., { mode: "cloud" }): dates on, no
+            // allowlist, no tokens. What is sent is what cloud mode would send.
+            const local = engine.redact(text, { countries, detectDates: true, cache: false });
+            const [masked] = maskForCloud(text, local.detections);
+            const line = { custom_id: customId, text: masked, countries };
+            if (language)
+                line.language = language;
+            lines.push(JSON.stringify(line));
+            const cp = utf16ToCodePoints(text);
+            entries[customId] = {
+                text: text,
+                masked_sha256: sha256Hex([masked]),
+                countries,
+                language,
+                detections: local.detections.map(d => toStored(d, cp)),
+                detection_mode: local.detectionMode,
+                inferred_countries: local.inferredCountries,
+            };
+        }
+        const body = encoder.encode(lines.join("\n") + "\n");
+        if (body.length > MAX_BODY_BYTES) {
+            throw new BatchError(`the masked batch is ${body.length.toLocaleString("en-US")} bytes; the limit is ` +
+                `${MAX_BODY_BYTES.toLocaleString("en-US")}`);
+        }
+        const compressed = gzip(body);
+        const headers = {
+            "Content-Type": "application/x-ndjson",
+            "Idempotency-Key": options.idempotencyKey ?? crypto.randomUUID(),
+        };
+        if (compressed !== null)
+            headers["Content-Encoding"] = "gzip";
+        const resp = await this.request("POST", "/v1/batches", compressed ?? body, headers);
+        const payload = await this.json(resp);
+        if (resp.status !== 200 && resp.status !== 201)
+            this.raiseFor(resp.status, payload);
+        const batch = Batches.batch(payload);
+        if (!batch.id)
+            throw new BatchError("the gateway accepted the batch but returned no id");
+        await this.save({
+            format: FILE_FORMAT,
+            batch_id: batch.id,
+            status: "pending",
+            created_at: batch.createdAt ?? now(),
+            expires_at: batch.expiresAt,
+            results_expire_at: null,
+            resolved_at: null,
+            engine: "euredact-ts",
+            engine_version: null,
+            counts: { documents: Object.keys(entries).length },
+            entries,
+        });
+        return batch;
+    }
+    // -- retrieve, cancel, pending, purge, sweep ------------------------------
+    /** The gateway's view: status, counts and timestamps. */
+    async retrieve(batchId) {
+        await this.sweep();
+        const resp = await this.request("GET", `/v1/batches/${encodeURIComponent(batchId)}`);
+        const payload = await this.json(resp);
+        if (resp.status !== 200)
+            this.raiseFor(resp.status, payload);
+        const batch = Batches.batch(payload);
+        const state = await this.load(batchId);
+        if (state !== null && batch.resultsExpireAt && state.results_expire_at !== batch.resultsExpireAt) {
+            state.results_expire_at = batch.resultsExpireAt;
+            await this.save(state);
+        }
+        return batch;
+    }
+    /**
+     * This account's batches, newest first (1-100, the gateway's bound). The
+     * gateway's view only: `pending()` lists the ones with a local file here.
+     */
+    async list(limit = 20) {
+        if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
+            throw new RangeError("limit must be between 1 and 100");
+        }
+        const resp = await this.request("GET", `/v1/batches?limit=${limit}`);
+        const payload = await this.json(resp);
+        if (resp.status !== 200)
+            this.raiseFor(resp.status, payload);
+        const raw = Array.isArray(payload.batches) ? payload.batches : [];
+        return raw.filter((b) => !!b && typeof b === "object" && !Array.isArray(b))
+            .map(b => Batches.batch(b));
+    }
+    /** Cancel queued documents; the local file stays pending until the finished
+     *  ones are mapped by `results()`. */
+    async cancel(batchId) {
+        const resp = await this.request("POST", `/v1/batches/${encodeURIComponent(batchId)}/cancel`);
+        const payload = await this.json(resp);
+        if (resp.status !== 200 && resp.status !== 202)
+            this.raiseFor(resp.status, payload);
+        return Batches.batch(payload);
+    }
+    /** Batch ids whose local file still waits to be mapped. Reads files only. */
+    async pending() {
+        await this.sweep();
+        const out = [];
+        for (const id of await this.store.list()) {
+            const state = await this.load(id);
+            if (state !== null && state.status === "pending")
+                out.push(id);
+        }
+        return out;
+    }
+    /** Delete the local file, whatever its status. */
+    async purge(batchId) {
+        await this.store.delete(batchId);
+    }
+    /**
+     * Delete receipts and expire pending files past `results_expire_at`. Runs at
+     * the start of the calls that look in the store, so nothing has to be
+     * scheduled. A pending file past that time can never be mapped -- the gateway
+     * has swept the results -- so its originals are wiped.
+     */
+    async sweep(at = new Date()) {
+        for (const id of await this.store.list()) {
+            let state;
+            try {
+                state = await this.load(id);
+            }
+            catch {
+                continue;
+            }
+            if (state === null || !state.results_expire_at)
+                continue;
+            if (at < new Date(state.results_expire_at))
+                continue;
+            if (state.status === "pending")
+                await this.wipe(state, "expired");
+            else
+                await this.store.delete(id);
+        }
+    }
+    async wipe(state, status) {
+        state.entries = {};
+        state.status = status;
+        state.resolved_at = now();
+        await this.save(state);
+    }
+    // -- results ----------------------------------------------------------
+    /**
+     * Map an ended batch back onto the originals in its local file. Each
+     * document's masked text is rebuilt from the file and must hash to both the
+     * stored SHA-256 and the gateway's `masked_sha256` before a span is placed.
+     * Anything that cannot be mapped is a per-document error, never the masked
+     * text passed off as a result.
+     */
+    async results(batchId) {
+        const state = await this.load(batchId);
+        if (state !== null && (state.status === "resolved" || state.status === "expired")) {
+            return {
+                batchId, documents: {}, batch: null,
+                status: state.status === "resolved" ? "already_resolved" : "expired",
+            };
+        }
+        const batch = await this.retrieve(batchId);
+        if (batch.status !== "ended")
+            return { batchId, status: "not_ended", documents: {}, batch };
+        const resp = await this.request("GET", `/v1/batches/${encodeURIComponent(batchId)}/results`);
+        if (resp.status === 410) {
+            if (state !== null)
+                await this.wipe(state, "expired");
+            return { batchId, status: "expired", documents: {}, batch };
+        }
+        if (resp.status === 409)
+            return { batchId, status: "not_ended", documents: {}, batch };
+        if (resp.status !== 200)
+            this.raiseFor(resp.status, await this.json(resp));
+        const entries = state?.entries ?? {};
+        const documents = {};
+        for (const line of (await resp.text()).split("\n")) {
+            if (!line.trim())
+                continue;
+            const row = JSON.parse(line);
+            const customId = row.custom_id ?? "";
+            documents[customId] = this.outcome(customId, row.result ?? {}, entries[customId], state !== null);
+        }
+        if (state !== null) {
+            const counts = {
+                documents: Object.keys(documents).length,
+                mapped: Object.values(documents).filter(o => o.ok).length,
+            };
+            for (const o of Object.values(documents)) {
+                if (o.error)
+                    counts[o.error] = (counts[o.error] ?? 0) + 1;
+            }
+            state.counts = counts;
+            await this.wipe(state, "resolved");
+        }
+        return { batchId, status: "resolved", documents, batch };
+    }
+    outcome(customId, result, entry, fileExists) {
+        const fail = (error, message) => ({ customId, ok: false, result: null, error, message });
+        if (result.type !== "succeeded") {
+            const err = result.error ?? {};
+            const code = err.code ?? result.type ?? "errored";
+            let message = err.message ?? "";
+            if (code === "too_long") {
+                message = `${message}. The ${MAX_DOCUMENT_TOKENS.toLocaleString("en-US")}-token limit is ` +
+                    "counted by the model's tokenizer on the gateway; the SDK cannot check it before " +
+                    "upload. Split the document and submit the parts.";
+                message = message.replace(/^\. /, "");
+            }
+            return fail(code, message);
+        }
+        if (entry === undefined) {
+            return fail("missing_local_entry", fileExists
+                ? "no entry for this document in the local batch file"
+                : "no local batch file on this machine");
+        }
+        const text = entry.text;
+        const unit = codePointOffsets(text);
+        const local = entry.detections.map(d => fromStored(d, unit));
+        const [masked, labels] = maskForCloud(text, local);
+        if (sha256Hex([masked]) !== entry.masked_sha256 || result.masked_sha256 !== entry.masked_sha256) {
+            return fail("local_mismatch", "the local batch file does not match what was sent");
+        }
+        // The gateway's offsets count code points of the masked text; toResult
+        // converts them to UTF-16 units, as the cloud client does for redactAsync.
+        const remote = toResult({ entities: result.entities }, masked);
+        let placed;
+        try {
+            placed = ontoOriginal(remote.detections, masked, text, labels, CloudError);
+        }
+        catch (err) {
+            return fail("local_mismatch", err.message);
+        }
+        const detections = [...local, ...placed].sort((a, b) => a.start - b.start || b.end - a.end);
+        return {
+            customId, ok: true, error: null, message: "",
+            result: {
+                redactedText: applyReplacements(text, detections, det => `[${det.entityType}]`),
+                detections,
+                source: "cloud",
+                degraded: false,
+                inferredCountries: entry.inferred_countries,
+                evidence: [],
+                detectionMode: entry.detection_mode,
+                tokens: {},
+                exempted: [],
+                // A batch line carries the model and anything unplaced; it has no job
+                // id and no usage of its own (the batch carries those).
+                cloud: toCloudInfo({ model_version: result.model_version, unlocated: result.unlocated }),
+            },
+        };
+    }
+}

package/dist/esm/cloud/client.d.ts

@@ -25,7 +25,33 @@
  * own `fetch`.
  */
 import { type CloudConfig } from "./config.js";
-import { type RedactResult } from "../types.js";
+import { CloudError } from "./errors.js";
+import { type CloudInfo, type RedactResult, type Usage } from "../types.js";
+/** @internal Shared with cloud/account.ts. */
+export declare const RETRY_STATUS: Set<number>;
+/** One span as the service reports it. */
+interface WireSpan {
+    start: number;
+    end: number;
+    text: string;
+    type: string;
+    source: string;
+    match?: string;
+    confidence?: string;
+}
+interface WireResult {
+    job_id?: string;
+    status?: string;
+    redacted_text?: string;
+    entities?: WireSpan[];
+    unlocated?: unknown;
+    model_version?: string | null;
+    stats?: Record<string, unknown>;
+    usage?: unknown;
+    error?: string;
+    detail?: Record<string, unknown>;
+    location?: string;
+}
 export interface CloudRedactOptions {
     country: string;
     language?: string;
@@ -34,6 +60,59 @@ export interface CloudRedactOptions {
     /** Injectable for tests. Defaults to the platform `fetch`. */
     fetchImpl?: typeof fetch;
 }
+/** @internal */
+export declare function requireFetch(supplied?: typeof fetch): typeof fetch;
+/** @internal */
+export declare function sleep(ms: number): Promise<void>;
+/** @internal */
+export declare function retryAfterMs(headers: Headers | undefined): number | null;
+/**
+ * Full jitter. A synchronised retry storm from many clients is how a
+ * recovering service is knocked back over.
+ */
+/** @internal */
+export declare function backoffMs(attempt: number, retryAfter: number | null): number;
+/** @internal */
+export declare function uuid(): string;
+/**
+ * Map the service's offsets, which count code points, onto this SDK's, which
+ * count UTF-16 units.
+ *
+ * The two agree until the first character outside the BMP — an emoji, say —
+ * after which every service offset is short by one unit per such character.
+ * Returns the identity when the text has none, which is the common case and
+ * costs one regex test.
+ */
+/** @internal Shared with cloud/batches.ts. */
+export declare function codePointOffsets(text: string): (cp: number) => number;
+/** @internal */
+export declare function isInteger(value: unknown): value is number;
+/**
+ * The `usage` block, or undefined when absent or malformed. Shared rules with
+ * the Python SDK (`conformance/cloud_usage.json`): `tokens` and `credits` are
+ * integers, `billing_rate` a number, `factors` a list whose entries carry a
+ * string `code`. Anything else there drops the whole block rather than
+ * reporting a cost nobody can trust; unknown keys are ignored.
+ * @internal Exported for the conformance test.
+ */
+export declare function toUsage(raw: unknown): Usage | undefined;
+/**
+ * The service's account of a request beside its spans. Shared rules with the
+ * Python SDK (`conformance/cloud_result.json`): a field of the wrong type reads
+ * as absent, and an unlocated entry without string `text` is skipped.
+ * @internal
+ */
+export declare function toCloudInfo(payload: Record<string, unknown>): CloudInfo;
+/**
+ * A 429 from the gateway's daily quota rather than the edge's rate limit: the
+ * gateway answers in JSON with `detail.used` and `detail.limit`; nginx in HTML.
+ * @internal
+ */
+export declare function isQuota(payload: Record<string, unknown>): boolean;
+/** The error for a non-success answer. @internal Shared with batches and account. */
+export declare function cloudErrorFor(status: number, payload: Record<string, unknown>): CloudError;
+/** @internal Shared with cloud/batches.ts. */
+export declare function toResult(payload: WireResult, text: string): RedactResult;
 export declare class CloudClient {
     readonly config: CloudConfig;
     constructor(config?: CloudConfig | null);
@@ -43,3 +122,6 @@ export declare class CloudClient {
     private poll;
     private withTimeout;
 }
+/** @internal */
+export declare function readJson(response: Response): Promise<WireResult>;
+export {};