@hue-run/sdk 0.17.0 → 0.18.0

package/dist/evals/media.js

@@ -0,0 +1,907 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { isLoopbackHost } from "../config.js";
+import { uuid } from "./json.js";
+const messages = {
+    not_found: "Hue holds no file with this digest for the trace or snapshot",
+    missing: "The trace lists this file, but its bytes were never uploaded or have been erased",
+    unavailable: "Hue could not read the file now; retry later",
+    forbidden: "The key cannot read recorded content; use a Read or Read and write key",
+    too_large: "The file is larger than maxBytes",
+    digest_mismatch: "The downloaded bytes do not match the requested SHA-256",
+    failed: "The media request failed",
+};
+/** Thrown by {@link MediaClient.fetch}. The message is fixed and never includes response text. */
+export class MediaFetchError extends Error {
+    reason;
+    status;
+    retryAfterSeconds;
+    constructor(
+    /** What failed. */
+    reason, 
+    /** HTTP status of the answer that failed, when there was one. */
+    status, 
+    /** Seconds Hue asked the caller to wait, on `unavailable`, when it said. */
+    retryAfterSeconds, options) {
+        super(reason === "failed" && status ? `${messages.failed} (HTTP ${status})` : messages[reason], options);
+        this.reason = reason;
+        this.status = status;
+        this.retryAfterSeconds = retryAfterSeconds;
+        this.name = "MediaFetchError";
+    }
+}
+const SHA256 = /^[0-9a-f]{64}$/;
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const TRACE = /^[0-9a-f]{32}$/i;
+const SPAN = /^[0-9a-f]{16}$/i;
+const MEDIA_TYPE = /^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*\/[A-Za-z0-9!#$&^_.+-]+$/;
+/** The default and largest `maxBytes`: Hue keeps values of up to 1 GB. */
+const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
+const MAX_BYTES = 1_000_000_000;
+/** The least time a download may go without a byte before it ends: two minutes. Tests shorten it;
+ * it is not exported from the package. */
+export const mediaDownload = { minIdleMillis: 120_000 };
+/** The longest `Retry-After` the one retry of an unavailable read waits. */
+const MAX_RETRY_AFTER_SECONDS = 10;
+/** Bytes of a JSON answer read: a link answer or an error body. */
+const MAX_ANSWER_BYTES = 64 * 1024;
+/** What the SDKs keep of text media over 64 KiB in its content field: its first 16 KiB. */
+const PREFIX_BYTES = 16 * 1024;
+/** Nesting a value is walked to; deeper parts stay as they are. */
+const MAX_DEPTH = 128;
+function digestOf(value) {
+    const digest = typeof value === "string" ? value.toLowerCase() : value;
+    if (typeof digest !== "string" || !SHA256.test(digest))
+        throw new TypeError("sha256 must be 64 hex characters");
+    return digest;
+}
+function maxBytesOf(value) {
+    const bytes = value ?? DEFAULT_MAX_BYTES;
+    if (!Number.isSafeInteger(bytes) || bytes < 1 || bytes > MAX_BYTES)
+        throw new RangeError("maxBytes must be an integer from 1 to 1000000000");
+    return bytes;
+}
+/** The route of a file and the digest it names, validated before anything is sent. */
+function routeOf(target) {
+    if (!target || typeof target !== "object")
+        throw new TypeError("Name a traceId or snapshotId");
+    const sha256 = digestOf(target.sha256);
+    if (target.snapshotId !== undefined) {
+        if (target.traceId !== undefined || target.spanId !== undefined)
+            throw new TypeError("Name a traceId or a snapshotId, not both");
+        return {
+            path: `/trace-snapshots/${uuid(target.snapshotId).toLowerCase()}/blobs/${sha256}`,
+            sha256,
+        };
+    }
+    const traceId = target.traceId;
+    if (typeof traceId !== "string" ||
+        !(UUID.test(traceId) || (TRACE.test(traceId) && !/^0+$/.test(traceId))))
+        throw new TypeError("traceId must be a Hue trace ID or a 32-hex OpenTelemetry trace ID");
+    let query = "";
+    if (target.spanId !== undefined) {
+        if (typeof target.spanId !== "string" ||
+            !SPAN.test(target.spanId) ||
+            /^0+$/.test(target.spanId))
+            throw new TypeError("spanId must be a 16-hex OpenTelemetry span ID");
+        query = `?spanId=${target.spanId.toLowerCase()}`;
+    }
+    return { path: `/traces/${traceId.toLowerCase()}/blobs/${sha256}${query}`, sha256 };
+}
+/** The host an `http:` link writes, before the URL parser rewrites it: after the authority's last
+ * `@`, up to its port, the authority ending at the first `/`, `?` or `#`, as Python splits it. */
+const WRITTEN_HTTP_HOST = /^http:\/\/(?:[^/?#]*@)?(\[[^\]/?#]*\]|[^/?#:@]*)/i;
+/**
+ * A download link Hue issued: HTTPS, or HTTP to a loopback host written `localhost`, `127.0.0.1`
+ * or `[::1]`, without credentials or a fragment. A host the URL parser rewrites to one of those
+ * (`127.1`, `0x7f.0.0.1`, `[0::1]`) is refused, as the Python SDK refuses it.
+ */
+function linkOf(value) {
+    // eslint-disable-next-line no-control-regex -- control characters are refused
+    if (typeof value !== "string" || value.length > 8192 || /[\x00-\x20\x7f]/u.test(value))
+        return undefined;
+    try {
+        const url = new URL(value);
+        const allowed = url.protocol === "https:" ||
+            (url.protocol === "http:" &&
+                isLoopbackHost(url.hostname) &&
+                WRITTEN_HTTP_HOST.exec(value)?.[1]?.toLowerCase() === url.hostname);
+        return allowed && !url.username && !url.password && !url.hash ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The whole seconds of a `Retry-After` header, up to a day, when it is one. */
+function retryAfterOf(response) {
+    const header = response.headers.get("retry-after")?.trim() ?? "";
+    return /^\d{1,6}$/.test(header) ? Math.min(Number(header), 86_400) : undefined;
+}
+/** Waits `millis`, or less when `signal` ends first. */
+function pause(millis, signal) {
+    return new Promise((resolve) => {
+        const done = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", done);
+            resolve();
+        };
+        const timer = setTimeout(done, millis);
+        signal?.addEventListener("abort", done, { once: true });
+        if (signal?.aborted)
+            done();
+    });
+}
+/** The connection failed while an answer's body was read: no answer, unlike a malformed one. */
+class AnswerReadError extends Error {
+}
+/** A JSON answer read to a bound; undefined when it is larger or not JSON. Throws
+ * {@link AnswerReadError} when the connection fails while it is read. */
+async function answerOf(response) {
+    const reader = response.body?.getReader();
+    if (!reader)
+        return undefined;
+    const chunks = [];
+    let size = 0;
+    try {
+        for (;;) {
+            let read;
+            try {
+                read = await reader.read();
+            }
+            catch (error) {
+                throw new AnswerReadError("The answer could not be read", { cause: error });
+            }
+            if (read.done)
+                break;
+            size += read.value.byteLength;
+            if (size > MAX_ANSWER_BYTES)
+                return undefined;
+            chunks.push(read.value);
+        }
+        try {
+            return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        }
+        catch {
+            return undefined;
+        }
+    }
+    finally {
+        await reader.cancel().catch(() => undefined);
+    }
+}
+function grantOf(answer, sha256) {
+    const fields = answer !== null && typeof answer === "object" && !Array.isArray(answer)
+        ? answer
+        : undefined;
+    const url = linkOf(fields?.url);
+    const size = fields?.size;
+    if (!fields ||
+        fields.sha256 !== sha256 ||
+        url === undefined ||
+        typeof size !== "number" ||
+        !Number.isSafeInteger(size) ||
+        size < 0 ||
+        (fields.method !== undefined && fields.method !== "GET"))
+        throw new MediaFetchError("failed");
+    const contentType = typeof fields.contentType === "string" &&
+        fields.contentType.length <= 255 &&
+        MEDIA_TYPE.test(fields.contentType.split(";")[0].trim())
+        ? fields.contentType
+        : undefined;
+    const fileName = typeof fields.fileName === "string" &&
+        fields.fileName.length > 0 &&
+        fields.fileName.length <= 255
+        ? fields.fileName
+        : undefined;
+    return {
+        url,
+        size,
+        ...(contentType !== undefined ? { contentType } : {}),
+        ...(fileName !== undefined ? { fileName } : {}),
+    };
+}
+/**
+ * Hue's media reads, `client.media`: one recorded file by digest (`fetch`), and every digest part
+ * of a recorded value written back with its bytes (`resolve`). The project key is sent to Hue
+ * only; the file itself comes from a short-lived link fetched without it.
+ */
+export class MediaClient {
+    #baseUrl;
+    #apiKey;
+    #timeoutMillis;
+    /** @internal Created by the evaluation client as its `media`. */
+    constructor(options) {
+        this.#baseUrl = options.baseUrl;
+        this.#apiKey = options.apiKey;
+        this.#timeoutMillis = options.timeoutMillis;
+    }
+    /**
+     * Downloads one recorded file and checks it against its digest: from a trace (Hue reads the
+     * span's record that lists the digest under `hue.blobs`) or from a snapshot's pinned copy. A
+     * read Hue answers as unavailable (503 or 429) is asked once more after its `Retry-After`, at
+     * most 10 seconds, and so is one that lost its connection or timed out.
+     *
+     * @throws MediaFetchError for a file Hue does not hold, a refused key, an unavailable store, a
+     * file over `maxBytes`, bytes that do not match the digest or a failed request.
+     * @throws TypeError for an invalid trace, span or snapshot ID or digest.
+     */
+    async fetch(target, options = {}) {
+        const { path, sha256 } = routeOf(target);
+        const maxBytes = maxBytesOf(options.maxBytes);
+        const grant = await this.#grant(path, sha256, options.signal);
+        if (grant.size > maxBytes)
+            throw new MediaFetchError("too_large");
+        const bytes = await this.#download(grant, sha256, options.signal);
+        return {
+            bytes,
+            sha256,
+            size: bytes.byteLength,
+            ...(grant.contentType !== undefined ? { contentType: grant.contentType } : {}),
+            ...(grant.fileName !== undefined ? { fileName: grant.fileName } : {}),
+        };
+    }
+    /**
+     * Writes the bytes of every digest part in `value` back into its content field, the reverse of
+     * the SDK's media capture: each distinct digest is downloaded once with {@link fetch}, at most
+     * `concurrency` at a time. Returns a new value; a part whose file cannot be read stays as it is
+     * and is reported in `failures`. Parse attributes recorded as JSON text (such as
+     * `gen_ai.input.messages`) before passing them; pass the record's `hue.blobs` as `blobs` when
+     * reading a snapshot, whose copies carry no media type.
+     *
+     * @throws TypeError for invalid identifiers or options; never for one file's failure.
+     */
+    async resolve(value, options) {
+        if (!options || typeof options !== "object")
+            throw new TypeError("Name a traceId or snapshotId");
+        const as = writeModeOf(options.as);
+        const refs = blobRefsOf(options.blobs);
+        const listed = listedTypes(refs);
+        const concurrency = options.concurrency ?? 4;
+        if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16)
+            throw new RangeError("concurrency must be 1–16");
+        const maxBytes = maxBytesOf(options.maxBytes);
+        const target = (sha256) => options.snapshotId !== undefined
+            ? { snapshotId: options.snapshotId, sha256 }
+            : {
+                traceId: options.traceId,
+                sha256,
+                ...(options.spanId !== undefined ? { spanId: options.spanId } : {}),
+            };
+        // Identifiers are checked once, before any request.
+        routeOf(target("0".repeat(64)));
+        const parts = digestParts(value);
+        const prefixes = prefixFailures(value, refs);
+        const digests = [...new Set([...parts.values()].map((part) => part.sha256))];
+        const files = new Map();
+        let next = 0;
+        const worker = async () => {
+            for (let index = next++; index < digests.length; index = next++) {
+                const sha256 = digests[index];
+                try {
+                    const file = await this.fetch(target(sha256), { maxBytes, signal: options.signal });
+                    const listedType = file.contentType === undefined ? listed.get(sha256) : undefined;
+                    files.set(sha256, listedType === undefined ? file : { ...file, contentType: listedType });
+                }
+                catch (error) {
+                    files.set(sha256, error instanceof MediaFetchError
+                        ? error
+                        : new MediaFetchError("failed", undefined, undefined, { cause: error }));
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(concurrency, digests.length) }, worker));
+        const resolution = writeBack(value, parts, (sha256) => {
+            const file = files.get(sha256);
+            return file instanceof MediaFetchError ? file.reason : file;
+        }, as);
+        resolution.failures.push(...prefixes);
+        return resolution;
+    }
+    /** The download link of one file, asked once more after an unavailable answer or a lost
+     * connection. */
+    async #grant(path, sha256, signal) {
+        for (let attempt = 0;; attempt++) {
+            /** A lost connection, before or while the answer is read: asked once more, then failed. */
+            const lost = async (error) => {
+                if (attempt > 0 || signal?.aborted)
+                    throw new MediaFetchError("failed", undefined, undefined, { cause: error });
+                await pause(250 + Math.random() * 250, signal);
+            };
+            let response;
+            try {
+                response = await fetch(`${this.#baseUrl}/api/v1${path}`, {
+                    headers: { Authorization: `Bearer ${this.#apiKey}` },
+                    // A redirect is never followed: it is an unexpected answer, failed with its status
+                    // and not asked again, as a lost connection is.
+                    redirect: "manual",
+                    signal: AbortSignal.any([
+                        AbortSignal.timeout(this.#timeoutMillis),
+                        ...(signal ? [signal] : []),
+                    ]),
+                });
+            }
+            catch (error) {
+                await lost(error);
+                continue;
+            }
+            let answer;
+            const status = response.status;
+            if (status === 503 || status === 429) {
+                await response.body?.cancel().catch(() => undefined);
+                const seconds = retryAfterOf(response);
+                const wait = seconds ?? 1;
+                if (attempt > 0 || wait > MAX_RETRY_AFTER_SECONDS || signal?.aborted)
+                    throw new MediaFetchError("unavailable", status, seconds);
+                await pause(wait * 1000, signal);
+                continue;
+            }
+            if (status === 404 || response.ok) {
+                try {
+                    answer = await answerOf(response);
+                }
+                catch (error) {
+                    if (!(error instanceof AnswerReadError))
+                        throw error;
+                    await lost(error);
+                    continue;
+                }
+            }
+            if (status === 404) {
+                const missing = answer !== null &&
+                    typeof answer === "object" &&
+                    answer.state === "missing";
+                throw new MediaFetchError(missing ? "missing" : "not_found", status);
+            }
+            if (status === 401 || status === 403) {
+                await response.body?.cancel().catch(() => undefined);
+                throw new MediaFetchError("forbidden", status);
+            }
+            if (!response.ok) {
+                await response.body?.cancel().catch(() => undefined);
+                throw new MediaFetchError("failed", status);
+            }
+            return grantOf(answer, sha256);
+        }
+    }
+    /** The bytes behind a link, fetched without the key, stopped one byte past the reported size
+     * and checked against the digest. The download ends once no byte has arrived for the client's
+     * timeout, at least two minutes; a large file that keeps arriving is never cut off. */
+    async #download(grant, sha256, signal) {
+        const idleMillis = Math.max(this.#timeoutMillis, mediaDownload.minIdleMillis);
+        const idle = new AbortController();
+        let timer;
+        const restart = () => {
+            clearTimeout(timer);
+            timer = setTimeout(() => idle.abort(new DOMException("No byte arrived in time", "TimeoutError")), idleMillis);
+        };
+        restart();
+        let reader;
+        try {
+            let response;
+            try {
+                response = await fetch(grant.url, {
+                    redirect: "manual",
+                    signal: AbortSignal.any([idle.signal, ...(signal ? [signal] : [])]),
+                });
+            }
+            catch (error) {
+                throw new MediaFetchError("failed", undefined, undefined, { cause: error });
+            }
+            if (!response.ok) {
+                await response.body?.cancel().catch(() => undefined);
+                throw new MediaFetchError("failed", response.status);
+            }
+            // The answer arrived: its body has the same time again for each piece.
+            restart();
+            reader = response.body?.getReader();
+            const hash = createHash("sha256");
+            const bytes = new Uint8Array(grant.size);
+            let size = 0;
+            try {
+                if (reader)
+                    for (;;) {
+                        const { done, value } = await reader.read();
+                        if (done)
+                            break;
+                        restart();
+                        if (size + value.byteLength > grant.size)
+                            throw new MediaFetchError("digest_mismatch");
+                        bytes.set(value, size);
+                        hash.update(value);
+                        size += value.byteLength;
+                    }
+            }
+            catch (error) {
+                if (error instanceof MediaFetchError)
+                    throw error;
+                throw new MediaFetchError("failed", undefined, undefined, { cause: error });
+            }
+            if (size !== grant.size || hash.digest("hex") !== sha256)
+                throw new MediaFetchError("digest_mismatch");
+            return bytes;
+        }
+        finally {
+            clearTimeout(timer);
+            await reader?.cancel().catch(() => undefined);
+        }
+    }
+}
+const BLOB_CONTENT_TYPE = /^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*\/[A-Za-z0-9!#$&^_.+-]+(?: *;[ -~]*)?$/;
+/**
+ * The entries a record's `hue.blobs` lists, read as Hue's reader reads them (`blobRefsOf`): an
+ * array of compact JSON object strings, of objects, or one string holding the JSON array; at most
+ * 64, one per key, the first winning. An entry without a valid key, digest and size is ignored, and
+ * so is anything that is no list.
+ */
+function blobRefsOf(listed) {
+    let entries = listed;
+    if (typeof entries === "string") {
+        try {
+            entries = JSON.parse(entries);
+        }
+        catch {
+            return [];
+        }
+    }
+    if (!Array.isArray(entries))
+        return [];
+    const refs = [];
+    const keys = new Set();
+    for (const item of entries.slice(0, 64)) {
+        let entry = item;
+        if (typeof entry === "string") {
+            if (entry.length > 2048)
+                continue;
+            try {
+                entry = JSON.parse(entry);
+            }
+            catch {
+                continue;
+            }
+        }
+        const fields = entry !== null && typeof entry === "object" && !Array.isArray(entry)
+            ? entry
+            : undefined;
+        if (!fields)
+            continue;
+        const key = fields.key;
+        const sha256 = typeof fields.sha256 === "string" ? fields.sha256.toLowerCase() : "";
+        const size = typeof fields.size === "number"
+            ? fields.size
+            : typeof fields.size === "string" && /^\d{1,10}$/.test(fields.size)
+                ? Number(fields.size)
+                : NaN;
+        if (typeof key !== "string" ||
+            key.length < 1 ||
+            key.length > 256 ||
+            // eslint-disable-next-line no-control-regex -- control characters are refused
+            /[\u0000-\u001f\u007f]/.test(key) ||
+            keys.has(key) ||
+            !SHA256.test(sha256) ||
+            !Number.isSafeInteger(size) ||
+            size < 1 ||
+            size > 1_000_000_000)
+            continue;
+        keys.add(key);
+        const type = fields.content_type ?? fields.contentType;
+        refs.push({
+            key,
+            sha256,
+            size,
+            ...(typeof type === "string" &&
+                type.length >= 3 &&
+                type.length <= 255 &&
+                BLOB_CONTENT_TYPE.test(type)
+                ? { contentType: type }
+                : {}),
+        });
+    }
+    return refs;
+}
+/** The media type each digest has in a record's `hue.blobs` entries, the first typed one of a
+ * digest. */
+function listedTypes(refs) {
+    const types = new Map();
+    for (const ref of refs)
+        if (ref.contentType !== undefined && !types.has(ref.sha256))
+            types.set(ref.sha256, ref.contentType);
+    return types;
+}
+/**
+ * The parts of `value` whose content field holds only the first 16 KiB of a file (text media over
+ * 64 KiB, as the SDKs record it), found by the `hue.blobs` entries that name that field
+ * (`<attribute>#<pointer to the field>`): reported as `prefix_only` with the entry's SHA-256, never
+ * written. An entry names a pointer, not the value, so entries of other attributes at the same
+ * pointer are reported too; a part in digest form, or whose field holds no such prefix, is left
+ * out.
+ */
+function prefixFailures(value, refs) {
+    const failures = [];
+    const seen = new Set();
+    for (const ref of refs) {
+        const at = ref.key.indexOf("#");
+        const pointer = at === -1 ? "" : ref.key.slice(at + 1);
+        if (!pointer.startsWith("/"))
+            continue;
+        const segments = pointer
+            .slice(1)
+            .split("/")
+            .map((segment) => segment.replaceAll("~1", "/").replaceAll("~0", "~"));
+        const nodes = [value];
+        for (const segment of segments) {
+            const node = nodes.at(-1);
+            const next = Array.isArray(node)
+                ? /^\d+$/.test(segment)
+                    ? node[Number(segment)]
+                    : undefined
+                : plain(node) && Object.hasOwn(node, segment)
+                    ? node[segment]
+                    : undefined;
+            if (next === undefined)
+                break;
+            nodes.push(next);
+        }
+        const recorded = nodes.at(-1);
+        if (nodes.length !== segments.length + 1 || typeof recorded !== "string")
+            continue;
+        // The SDKs keep the first 16 KiB of the content, cut back to a character.
+        const kept = Buffer.byteLength(recorded, "utf8");
+        if (kept > PREFIX_BYTES || kept <= PREFIX_BYTES - 4 || ref.size <= kept)
+            continue;
+        // The part is a few levels above its content field, as the field's slot names it.
+        for (let up = 1; up <= Math.min(3, segments.length); up++) {
+            const part = plain(nodes[nodes.length - 1 - up]);
+            const slot = part && slotOf(part);
+            if (!part || !slot || typeof part.sha256 === "string")
+                continue;
+            const path = [...slot.owner, slot.key];
+            const tail = segments.slice(-up);
+            if (path.length !== tail.length || path.some((segment, index) => segment !== tail[index]))
+                continue;
+            const partPointer = segments
+                .slice(0, segments.length - up)
+                .map((segment) => `/${segment.replaceAll("~", "~0").replaceAll("/", "~1")}`)
+                .join("");
+            const identity = `${partPointer}\u0000${ref.sha256}`;
+            if (!seen.has(identity)) {
+                seen.add(identity);
+                failures.push({ pointer: partPointer, sha256: ref.sha256, reason: "prefix_only" });
+            }
+            break;
+        }
+    }
+    return failures;
+}
+function writeModeOf(value) {
+    if (value === undefined)
+        return "dataUrl";
+    if (value !== "dataUrl" && value !== "bytes")
+        throw new TypeError('as must be "dataUrl" or "bytes"');
+    return value;
+}
+/**
+ * Writes case files back into the digest parts of a case's inputs, by SHA-256: the files a runner
+ * hands a target (`context.files`) or any bytes the caller holds, so a target can send the
+ * recorded model call again with the real image, document or audio. Each file is read once and
+ * checked against its digest; a part with no matching file, or whose file does not match, stays
+ * as it is and is reported in `failures`. Returns a new value; the input is never changed.
+ */
+export async function rehydrateMedia(value, files, options = {}) {
+    const as = writeModeOf(options.as);
+    if (!Array.isArray(files))
+        throw new TypeError("files must be a list of files");
+    const byDigest = new Map();
+    for (const file of files) {
+        const digest = typeof file?.sha256 === "string" ? file.sha256.toLowerCase() : undefined;
+        if (digest !== undefined && SHA256.test(digest) && !byDigest.has(digest))
+            byDigest.set(digest, file);
+    }
+    const parts = digestParts(value);
+    const read = new Map();
+    for (const sha256 of new Set([...parts.values()].map((part) => part.sha256))) {
+        const file = byDigest.get(sha256);
+        if (!file) {
+            read.set(sha256, "not_found");
+            continue;
+        }
+        let bytes;
+        try {
+            if (file.bytes !== undefined && !(file.bytes instanceof Uint8Array))
+                throw new TypeError("bytes must be a Uint8Array");
+            bytes = file.bytes ?? (await readFile(file.path));
+        }
+        catch {
+            read.set(sha256, "failed");
+            continue;
+        }
+        if (createHash("sha256").update(bytes).digest("hex") !== sha256) {
+            read.set(sha256, "digest_mismatch");
+            continue;
+        }
+        read.set(sha256, {
+            bytes,
+            sha256,
+            size: bytes.byteLength,
+            ...(file.contentType !== undefined ? { contentType: file.contentType } : {}),
+        });
+    }
+    return writeBack(value, parts, (sha256) => read.get(sha256), as);
+}
+/** A plain object, the only kind of object the walk copies and writes into. */
+function plain(value) {
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null ? value : undefined;
+}
+function text(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function declaredType(part) {
+    return text(part.mime_type ?? part.mediaType ?? part.media_type ?? part.mimeType);
+}
+const AUDIO_FORMATS = {
+    mp3: "audio/mpeg",
+    mpeg: "audio/mpeg",
+    mpga: "audio/mpeg",
+    wav: "audio/wav",
+    wave: "audio/wav",
+    ogg: "audio/ogg",
+    oga: "audio/ogg",
+    opus: "audio/opus",
+    flac: "audio/flac",
+    aac: "audio/aac",
+    m4a: "audio/mp4",
+    mp4: "audio/mp4",
+    webm: "audio/webm",
+    pcm16: "audio/pcm",
+    pcm: "audio/pcm",
+};
+const BEDROCK_TYPES = {
+    image: {
+        png: "image/png",
+        jpeg: "image/jpeg",
+        jpg: "image/jpeg",
+        gif: "image/gif",
+        webp: "image/webp",
+    },
+    document: {
+        pdf: "application/pdf",
+        csv: "text/csv",
+        doc: "application/msword",
+        docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        xls: "application/vnd.ms-excel",
+        xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        html: "text/html",
+        txt: "text/plain",
+        md: "text/markdown",
+    },
+};
+const audioType = (format) => typeof format === "string" ? AUDIO_FORMATS[format.trim().toLowerCase()] : undefined;
+/** The content slot of a part read by its shape, the shapes Hue reads media parts in; undefined
+ * for text, tool and reference parts (a provider file ID, a remote URL). */
+function slotOf(part) {
+    const type = typeof part.type === "string" ? part.type.toLowerCase().replaceAll("-", "_") : undefined;
+    const reference = typeof part.file_id === "string";
+    switch (type) {
+        case "blob":
+            return {
+                owner: [],
+                key: "content",
+                encoding: "base64",
+                mediaType: declaredType(part),
+            };
+        case "file": {
+            const file = plain(part.file);
+            if (file)
+                return typeof file.file_id === "string"
+                    ? undefined
+                    : {
+                        owner: ["file"],
+                        key: "file_data",
+                        encoding: "dataUrl",
+                        mediaType: declaredType(part),
+                    };
+            return reference
+                ? undefined
+                : { owner: [], key: "data", encoding: "dataUrl", mediaType: declaredType(part) };
+        }
+        case "image":
+        case "document": {
+            const source = plain(part.source);
+            // A source without a type, absent or null, is read as base64, as Hue reads it.
+            if (source)
+                return source.type === undefined || source.type === null || source.type === "base64"
+                    ? {
+                        owner: ["source"],
+                        key: "data",
+                        encoding: "base64",
+                        mediaType: text(source.media_type) ?? declaredType(part),
+                    }
+                    : undefined;
+            return type === "image"
+                ? { owner: [], key: "image", encoding: "dataUrl", mediaType: declaredType(part) }
+                : undefined;
+        }
+        case "image_url":
+            return plain(part.image_url)
+                ? { owner: ["image_url"], key: "url", encoding: "dataUrl", mediaType: declaredType(part) }
+                : { owner: [], key: "image_url", encoding: "dataUrl", mediaType: declaredType(part) };
+        case "input_image":
+        case "computer_screenshot":
+            return reference
+                ? undefined
+                : { owner: [], key: "image_url", encoding: "dataUrl", mediaType: declaredType(part) };
+        case "input_audio": {
+            const audio = plain(part.input_audio);
+            return audio
+                ? {
+                    owner: ["input_audio"],
+                    key: "data",
+                    encoding: "base64",
+                    mediaType: audioType(audio.format),
+                }
+                : {
+                    owner: [],
+                    key: "data",
+                    encoding: "base64",
+                    mediaType: audioType(part.format) ?? declaredType(part),
+                };
+        }
+        case "input_file":
+            return reference || typeof part.file_url === "string"
+                ? undefined
+                : { owner: [], key: "file_data", encoding: "dataUrl", mediaType: declaredType(part) };
+        case "media":
+            return {
+                owner: [],
+                key: "data",
+                encoding: "base64",
+                mediaType: declaredType(part),
+            };
+        case undefined: {
+            for (const key of ["inlineData", "inline_data"]) {
+                const data = plain(part[key]);
+                if (data)
+                    return {
+                        owner: [key],
+                        key: "data",
+                        encoding: "base64",
+                        mediaType: text(data.mimeType ?? data.mime_type),
+                    };
+            }
+            for (const key of ["image", "document"]) {
+                const block = plain(part[key]);
+                const source = plain(block?.source);
+                if (block && source && source.s3Location === undefined)
+                    return {
+                        owner: [key, "source"],
+                        key: "bytes",
+                        encoding: "base64",
+                        mediaType: typeof block.format === "string"
+                            ? BEDROCK_TYPES[key][block.format.toLowerCase()]
+                            : undefined,
+                    };
+            }
+            // An OpenAI Chat assistant message's spoken reply: the message itself is the part.
+            const audio = plain(part.audio);
+            if (audio && typeof audio.id === "string")
+                return {
+                    owner: ["audio"],
+                    key: "data",
+                    encoding: "base64",
+                    mediaType: audioType(audio.format),
+                };
+            return undefined;
+        }
+        default:
+            return undefined;
+    }
+}
+/** The slot of a part in digest form: a known shape that records a SHA-256 in place of its
+ * content, its content field absent or empty. */
+function digestSlotOf(part) {
+    if (typeof part.sha256 !== "string" || !SHA256.test(part.sha256))
+        return undefined;
+    const slot = slotOf(part);
+    if (!slot)
+        return undefined;
+    let owner = part;
+    for (const segment of slot.owner)
+        owner = plain(owner?.[segment]);
+    if (!owner)
+        return undefined;
+    const content = owner[slot.key];
+    return content === undefined || content === null || content === "" ? slot : undefined;
+}
+const escape = (key) => key.replaceAll("~", "~0").replaceAll("/", "~1");
+/** The digest parts of a value, by the part object, each once, in reading order. */
+function digestParts(value) {
+    const found = new Map();
+    const visit = (node, pointer, depth) => {
+        if (depth > MAX_DEPTH)
+            return;
+        if (Array.isArray(node)) {
+            node.forEach((item, index) => visit(item, `${pointer}/${index}`, depth + 1));
+            return;
+        }
+        const part = plain(node);
+        if (!part)
+            return;
+        const slot = digestSlotOf(part);
+        // The slot's owners must sit within the depth the copy is made to.
+        if (slot && depth + slot.owner.length < MAX_DEPTH && !found.has(part))
+            found.set(part, { pointer, sha256: part.sha256, slot });
+        // A message whose `audio` is a part still holds the parts of its `content`.
+        for (const [key, item] of Object.entries(part))
+            visit(item, `${pointer}/${escape(key)}`, depth + 1);
+    };
+    visit(value, "", 0);
+    return found;
+}
+/** The type a `data:` URL declares: the part's own, else the file's, without parameters. */
+function dataUrlType(slot, file) {
+    for (const candidate of [slot.mediaType, file.contentType]) {
+        const type = candidate?.split(";")[0].trim();
+        if (type && MEDIA_TYPE.test(type))
+            return type.toLowerCase();
+    }
+    return "application/octet-stream";
+}
+/**
+ * A file's bytes as a slot's convention spells them: bare base64 or a `data:` URL. A GenAI `blob`
+ * or Vertex `media` part takes base64 for every media type, as the GenAI conventions define its
+ * content and as Hue's reader decodes content that reads as base64; raw text such as `test`
+ * would itself read as base64 and stand for other bytes.
+ */
+function spelled(slot, file) {
+    const base64 = Buffer.from(file.bytes).toString("base64");
+    return slot.encoding === "dataUrl" ? `data:${dataUrlType(slot, file)};base64,${base64}` : base64;
+}
+/** Sets an own enumerable property, a key named `__proto__` included, which assignment would take
+ * as the object's prototype instead. */
+function setOwn(target, key, value) {
+    Object.defineProperty(target, key, {
+        value,
+        writable: true,
+        enumerable: true,
+        configurable: true,
+    });
+}
+/** A copy of `value` with each digest part whose file was read holding its bytes. */
+function writeBack(value, parts, fileOf, as) {
+    const failures = [];
+    let resolved = 0;
+    const reported = new Set();
+    const visit = (node, depth) => {
+        if (depth > MAX_DEPTH)
+            return node;
+        if (Array.isArray(node))
+            return node.map((item) => visit(item, depth + 1));
+        const source = plain(node);
+        if (!source)
+            return node;
+        const part = parts.get(source);
+        const file = part && fileOf(part.sha256);
+        if (part && typeof file === "string" && !reported.has(source)) {
+            failures.push({ pointer: part.pointer, sha256: part.sha256, reason: file });
+            reported.add(source);
+        }
+        const copy = {};
+        for (const [key, item] of Object.entries(source))
+            setOwn(copy, key, visit(item, depth + 1));
+        if (part && file !== undefined && typeof file !== "string") {
+            // The owners along the slot are this copy's own objects: the walk copied every plain one.
+            let owner = copy;
+            for (const segment of part.slot.owner)
+                owner = owner[segment];
+            owner[part.slot.key] = as === "bytes" ? file.bytes : spelled(part.slot, file);
+            delete copy.sha256;
+            delete copy.size;
+            resolved++;
+        }
+        return copy;
+    };
+    const result = visit(value, 0);
+    return { value: result, resolved, failures };
+}