@hue-run/sdk 0.17.0 → 0.18.1

package/dist/evals/files.js

@@ -1,9 +1,10 @@
 import { createHash } from "node:crypto";
 import { constants } from "node:fs";
-import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises";
 import { extname, join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import { ArtifactSizeError, HueApiError } from "./client.js";
+import { uuid } from "./json.js";
 /** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
 export const targetFileRoles = [
     "source",
@@ -413,33 +414,260 @@ export async function uploadOutputFiles(client, executionId, files, save, timing
         if (file.artifactId)
             continue;
         const bytes = await verifiedBytes(file.path, file);
-        const reserved = await client.reserveArtifact({
+        file.artifactId = await publishArtifact(client, {
             idempotencyKey: `hue-sdk:${executionId}:${file.sha256}:${sha256(Buffer.from(file.filename)).slice(0, 16)}`,
             filename: file.filename,
             contentType: file.contentType,
             byteSize: file.byteSize,
             sha256: file.sha256,
-        });
-        let state = reserved.state;
-        if (state !== "ready") {
-            // A verification already running holds the uploaded bytes; Hue refuses another upload then.
-            if (reserved.copyState !== "acknowledged" && state !== "verifying") {
-                const upload = await client.requestArtifactUpload(reserved.id);
-                try {
-                    await client.uploadArtifactBytes(upload, bytes, file.contentType);
-                }
-                catch {
-                    // A lost staging acknowledgement or an immutable object already present can only be
-                    // settled by verified completion; never rewrite a final object here.
-                }
+        }, bytes, `Generated file ${file.filename}`, timing);
+        await save();
+    }
+}
+/** Reserve, stage and verify one artifact; its ID once Hue verified it. Replaying the key settles
+ * on the same artifact, including one Hue finished verifying after an earlier attempt stopped. */
+async function publishArtifact(client, declared, body, label, timing) {
+    const reserved = await client.reserveArtifact(declared);
+    let state = reserved.state;
+    if (state !== "ready") {
+        // A verification already running holds the uploaded bytes; Hue refuses another upload then.
+        if (reserved.copyState !== "acknowledged" && state !== "verifying") {
+            // A checked file is read again, whole, only now and only if it still has the bytes its
+            // check hashed: a file that changed sends nothing.
+            const bytes = body instanceof Uint8Array ? body : await checkedBytes(body);
+            const upload = await client.requestArtifactUpload(reserved.id);
+            try {
+                if (body instanceof Uint8Array)
+                    await client.uploadArtifactBytes(upload, bytes, declared.contentType);
+                // Sent from the write's own buffer a slice at a time, never copied whole.
+                else
+                    await client.uploadArtifactStream(upload, slicesOf(bytes), bytes.byteLength, declared.contentType);
+            }
+            catch {
+                // A lost staging acknowledgement or an immutable object already present can only be
+                // settled by verified completion; never rewrite a final object here.
             }
-            state = await settleArtifact(client, reserved.id, timing);
         }
-        if (state !== "ready")
-            throw new Error(`Generated file ${file.filename} was not verified by Hue (${state})`);
-        file.artifactId = reserved.id;
-        await save();
+        state = await settleArtifact(client, reserved.id, timing);
+    }
+    if (state !== "ready")
+        throw new Error(`${label} was not verified by Hue (${state})`);
+    return reserved.id;
+}
+/** How much of a case file given by path its check reads at once, to hash it. */
+const CASE_FILE_SLICE = 1024 * 1024;
+/** Bytes as a stream of 1 MiB views of them, made as the request reads them. */
+function slicesOf(bytes) {
+    let offset = 0;
+    return new ReadableStream({
+        pull(controller) {
+            if (offset >= bytes.byteLength)
+                return controller.close();
+            const end = Math.min(bytes.byteLength, offset + CASE_FILE_SLICE);
+            controller.enqueue(bytes.subarray(offset, end));
+            offset = end;
+        },
+    }, { highWaterMark: 0 });
+}
+/**
+ * The bytes of a checked case file, read whole when it is uploaded: case files are uploaded one
+ * at a time, into one buffer the write reuses, so no more than one is held in memory. Only bytes
+ * with the checked size and SHA-256 are returned, so the bytes sent are the bytes the check
+ * hashed; a file that changed after its check, or can no longer be read, is refused with a
+ * `TypeError` before anything is sent. The bytes are valid until the next file is read.
+ */
+async function checkedBytes(file) {
+    const changed = () => new TypeError(`${file.label} changed after it was checked`);
+    const unreadable = (error) => new TypeError(`${file.label} could not be read`, { cause: error });
+    const handle = await open(file.path, "r").catch((error) => {
+        throw unreadable(error);
+    });
+    try {
+        const read = (buffer, offset) => handle
+            .read(buffer, offset, buffer.byteLength - offset, null)
+            .then(({ bytesRead }) => bytesRead)
+            .catch((error) => {
+            throw unreadable(error);
+        });
+        // Never more than its checked size and one byte, the byte that shows it grew.
+        if ((file.scratch.buffer?.byteLength ?? 0) < file.byteSize)
+            file.scratch.buffer = Buffer.allocUnsafe(file.byteSize);
+        const bytes = file.scratch.buffer.subarray(0, file.byteSize);
+        for (let offset = 0; offset < bytes.byteLength;) {
+            const bytesRead = await read(bytes, offset);
+            if (!bytesRead)
+                throw changed();
+            offset += bytesRead;
+        }
+        if ((await read(Buffer.alloc(1), 0)) || sha256(bytes) !== file.sha256)
+            throw changed();
+        return bytes;
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}
+/** Roles a pinned case input can carry. */
+const caseFileRoles = [
+    ...targetFileRoles,
+    "org_template",
+    "evaluator_reference",
+];
+/** The most input files one case pins. */
+const MAX_CASE_FILES = 32;
+/** A case file name as the artifact API takes it: well-formed Unicode of 1–255 UTF-16 code units,
+ * as Hue counts them, without a control character or a path separator. */
+function caseFileName(name) {
+    if (typeof name !== "string" ||
+        !name.length ||
+        name.length > 255 ||
+        !name.isWellFormed() ||
+        [...name].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127 || c === "/" || c === "\\"))
+        throw new TypeError("A case file needs a well-formed filename of 1–255 UTF-16 code units without path separators or control characters");
+    return name;
+}
+/** A case file given by path, checked: a regular file within the size limit, hashed a slice at a
+ * time so it is never held whole. Its emptiness is the caller's check, as for bytes. */
+async function checkedFile(path, label, scratch) {
+    if (typeof path !== "string" || !path)
+        throw new TypeError(`${label} needs bytes or a path`);
+    // A path with no file behind it, or one that cannot be read, is refused as any invalid file.
+    const unreadable = (error) => new TypeError(`${label} could not be read`, { cause: error });
+    const info = await stat(path).catch((error) => {
+        throw unreadable(error);
+    });
+    if (!info.isFile())
+        throw new TypeError(`${label} is not a regular file`);
+    if (info.size > outputFileLimits.bytes)
+        throw new RangeError(`${label} exceeds 25 MiB`);
+    const handle = await open(path, "r").catch((error) => {
+        throw unreadable(error);
+    });
+    try {
+        const hash = createHash("sha256");
+        const slice = Buffer.allocUnsafe(CASE_FILE_SLICE);
+        let byteSize = 0;
+        for (;;) {
+            const { bytesRead } = await handle
+                .read(slice, 0, slice.byteLength, null)
+                .catch((error) => {
+                throw unreadable(error);
+            });
+            if (!bytesRead)
+                break;
+            byteSize += bytesRead;
+            // A file that grew past the limit since its size was read is refused, not read whole.
+            if (byteSize > outputFileLimits.bytes)
+                throw new RangeError(`${label} exceeds 25 MiB`);
+            hash.update(slice.subarray(0, bytesRead));
+        }
+        return { path, label, byteSize, sha256: hash.digest("hex"), scratch };
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}
+/** A stand-in for an artifact a case file is uploaded as, the length of a real ID. */
+const STAND_IN_ID = "00000000-0000-4000-8000-000000000000";
+/**
+ * The `files` of a case write as the case API takes them: references kept, and each file given
+ * as bytes or a path checked against the artifact policy (role, name, one of
+ * `outputContentTypes`, 1 byte to 25 MiB) and uploaded as a verified artifact. Every file is
+ * checked, and then `beforeUpload` is given the references with a stand-in ID for each upload,
+ * before the first upload. The reservation key names the version, the case's `externalKey`, the
+ * role and the bytes, so the same write sent again reuses its artifacts. Each artifact is pinned
+ * once with a role, even when a reference names the artifact a file was uploaded as.
+ */
+export async function caseFileReferences(client, versionId, input, beforeUpload = () => undefined, timing = artifactSettling) {
+    const files = input.files;
+    if (!Array.isArray(files))
+        throw new TypeError("files must be a list of case files");
+    if (files.length > MAX_CASE_FILES)
+        throw new RangeError(`A case pins at most ${MAX_CASE_FILES} files`);
+    const prepared = [];
+    const seen = new Set();
+    const scratch = {};
+    for (const [index, file] of files.entries()) {
+        const label = `Case file ${index + 1}`;
+        if (!file || typeof file !== "object")
+            throw new TypeError(`${label} is not a file`);
+        if (!caseFileRoles.includes(file.role))
+            throw new TypeError(`${label} has an unsupported role ${String(file.role)}`);
+        let identity;
+        if (file.artifactId !== undefined) {
+            // A reference keeps the artifact's own name and type; a file field beside it is refused
+            // rather than ignored.
+            const fields = file;
+            if (fields.filename !== undefined ||
+                fields.contentType !== undefined ||
+                fields.bytes !== undefined ||
+                fields.path !== undefined)
+                throw new TypeError(`${label} names an artifactId and a file; give one`);
+            const reference = { artifactId: uuid(file.artifactId).toLowerCase(), role: file.role };
+            prepared.push(reference);
+            identity = `artifact:${reference.artifactId}`;
+        }
+        else {
+            const filename = caseFileName(file.filename);
+            if (!outputContentTypes.includes(file.contentType))
+                throw new TypeError(`${label} has an unsupported content type ${String(file.contentType)}`);
+            if (file.bytes !== undefined && file.path !== undefined)
+                throw new TypeError(`${label} names bytes and a path; give one`);
+            if (file.bytes !== undefined && !(file.bytes instanceof Uint8Array))
+                throw new TypeError(`${label} bytes must be a Uint8Array`);
+            // Bytes are kept as given; a path is kept as its size and digest, and read again to upload.
+            const body = file.bytes ?? (await checkedFile(file.path, label, scratch));
+            const byteSize = body instanceof Uint8Array ? body.byteLength : body.byteSize;
+            if (!byteSize)
+                throw new RangeError(`${label} is empty`);
+            if (byteSize > outputFileLimits.bytes)
+                throw new RangeError(`${label} exceeds 25 MiB`);
+            const digest = body instanceof Uint8Array ? sha256(body) : body.sha256;
+            prepared.push({
+                role: file.role,
+                filename,
+                contentType: file.contentType,
+                byteSize,
+                sha256: digest,
+                body,
+            });
+            identity = `file:${digest}:${file.contentType}:${filename}`;
+        }
+        const pinned = `${identity}:${file.role}`;
+        if (seen.has(pinned))
+            throw new TypeError(`${label} is pinned twice with the role ${file.role}`);
+        seen.add(pinned);
+    }
+    beforeUpload(prepared.map((item) => "artifactId" in item ? item : { artifactId: STAND_IN_ID, role: item.role }));
+    const caseKey = sha256(Buffer.from(String(input.externalKey))).slice(0, 16);
+    const references = [];
+    // A reference and an upload that settle on the same artifact are pinned once: Hue refuses an
+    // artifact pinned twice with one role.
+    const pins = new Set();
+    const pin = (reference) => {
+        const identity = `${reference.artifactId.toLowerCase()}:${reference.role}`;
+        if (pins.has(identity))
+            return;
+        pins.add(identity);
+        references.push(reference);
+    };
+    for (const item of prepared) {
+        if ("artifactId" in item) {
+            pin(item);
+            continue;
+        }
+        const artifactId = await publishArtifact(client, {
+            // The key names everything the reservation declares, so a corrected name or type reserves
+            // a new artifact instead of colliding with the earlier one.
+            idempotencyKey: `hue-sdk:case:${versionId}:${caseKey}:${item.role}:${item.sha256}:${sha256(Buffer.from(item.filename)).slice(0, 16)}:${sha256(Buffer.from(item.contentType)).slice(0, 16)}`,
+            filename: item.filename,
+            contentType: item.contentType,
+            byteSize: item.byteSize,
+            sha256: item.sha256,
+        }, item.body, `Case file ${item.filename}`, timing);
+        pin({ artifactId, role: item.role });
     }
+    return references;
 }
 /** Project uploaded staged files as the verified local files handed to scorers. */
 export function localOutputFiles(files) {

package/dist/evals/media.d.ts

@@ -0,0 +1,221 @@
+/**
+ * Reading recorded media back. A trace records a large file as a digest part: the provider part
+ * with its content field removed and the file's `sha256` and `size` added, its bytes uploaded
+ * under the record's `hue.blobs`. `fetch` downloads one file by digest through a short-lived
+ * link Hue issues; `resolve` and {@link rehydrateMedia} put the bytes back into the parts of a
+ * recorded value, in the field and encoding each part's shape uses.
+ */
+/** Why a media read failed. */
+export type MediaFetchErrorReason = 
+/** Hue holds no such file for the trace or snapshot: an unknown trace, another project's, or a
+ * digest none of its records list (with `spanId`, that span's record). */
+"not_found"
+/** The trace lists the file, but its bytes were never uploaded or have been erased. */
+ | "missing"
+/** Hue could not read its evidence store, again after one retry; `retryAfterSeconds` says when
+ * to ask again. */
+ | "unavailable"
+/** The key cannot read recorded content: use a Read or Read and write key, and a Read and write
+ * key for a snapshot's copy. */
+ | "forbidden"
+/** The file is larger than `maxBytes`; nothing was downloaded. */
+ | "too_large"
+/** The downloaded bytes do not have the requested SHA-256 or the size Hue reported. */
+ | "digest_mismatch"
+/** A connection failure, a timeout, an unexpected status or an invalid response. */
+ | "failed";
+/** Thrown by {@link MediaClient.fetch}. The message is fixed and never includes response text. */
+export declare class MediaFetchError extends Error {
+    /** What failed. */
+    readonly reason: MediaFetchErrorReason;
+    /** HTTP status of the answer that failed, when there was one. */
+    readonly status?: number | undefined;
+    /** Seconds Hue asked the caller to wait, on `unavailable`, when it said. */
+    readonly retryAfterSeconds?: number | undefined;
+    constructor(
+    /** What failed. */
+    reason: MediaFetchErrorReason, 
+    /** HTTP status of the answer that failed, when there was one. */
+    status?: number | undefined, 
+    /** Seconds Hue asked the caller to wait, on `unavailable`, when it said. */
+    retryAfterSeconds?: number | undefined, 
+    /** `cause`, the error behind a failure; `snapshot`, set when a snapshot's copy was read, so a
+     * `forbidden` message names the Read and write key it needs. */
+    options?: {
+        cause?: unknown;
+        snapshot?: boolean;
+    });
+}
+/** One recorded file, downloaded and checked against its digest. */
+export interface MediaFile {
+    /** The file's bytes; their SHA-256 is `sha256`. */
+    bytes: Uint8Array;
+    /** Lowercase hex SHA-256 of `bytes`, as requested. */
+    sha256: string;
+    /** Byte length of `bytes`. */
+    size: number;
+    /** The media type the record lists for the file; absent for a snapshot's copy, whose type is
+     * the part's own. */
+    contentType?: string;
+    /** The recorded file name as Hue serves it; absent for a snapshot's copy. */
+    fileName?: string;
+}
+/** The file {@link MediaClient.fetch} reads: from a trace, or from a snapshot's pinned copy. */
+export type MediaFetchTarget = {
+    /** Hue trace ID (UUID) or the 32-hex OpenTelemetry trace ID. */
+    traceId: string;
+    /** Lowercase hex SHA-256 the part records. */
+    sha256: string;
+    /** 16-hex OpenTelemetry span ID of the span that recorded the part. Pass it when known: Hue
+     * then reads that span's record alone instead of searching the trace. */
+    spanId?: string;
+    /** Not set when reading a live trace's file. */
+    snapshotId?: undefined;
+} | {
+    /** Trace snapshot ID (UUID), such as the one a case converted from a trace keeps. Reading
+     * its copy needs a Read and write key. */
+    snapshotId: string;
+    /** Lowercase hex SHA-256 the part records. */
+    sha256: string;
+    /** Not set when reading a snapshot's pinned copy. */
+    traceId?: undefined;
+    /** Not set when reading a snapshot's pinned copy. */
+    spanId?: undefined;
+};
+/** Options for {@link MediaClient.fetch}. */
+export interface MediaFetchOptions {
+    /** Largest file read into memory, in bytes; a larger one fails with `too_large` before any byte
+     * is downloaded. Default 64 MiB, at most 1 GB. */
+    maxBytes?: number;
+    /** Ends the request, the retry wait and the download. */
+    signal?: AbortSignal;
+}
+/** Where {@link MediaClient.resolve} reads the files of a recorded value from, and how it writes
+ * them. */
+export type MediaResolveOptions = ({
+    /** Hue trace ID (UUID) or the 32-hex OpenTelemetry trace ID the value was recorded in. */
+    traceId: string;
+    /** 16-hex OpenTelemetry span ID of the span that recorded the value. */
+    spanId?: string;
+    /** Not set when resolving a live trace's files. */
+    snapshotId?: undefined;
+} | {
+    /** Trace snapshot ID whose pinned copies hold the files; reading them needs a Read and
+     * write key. */
+    snapshotId: string;
+    /** Not set when resolving a snapshot's pinned copies. */
+    traceId?: undefined;
+    /** Not set when resolving a snapshot's pinned copies. */
+    spanId?: undefined;
+}) & MediaWriteOptions & {
+    /** The `hue.blobs` entries of the record the value was read from, in any form Hue reads: an
+     * array of compact JSON object strings (what the SDKs write), an array of objects, or one
+     * string holding the JSON array. A part that declares no media type, read from a copy that
+     * carries none (a snapshot's), takes the type of the entry with its SHA-256 for its `data:`
+     * URL, rather than `application/octet-stream`. A part whose content field an entry names and
+     * that holds only the first 16 KiB of its file is reported as `prefix_only`. Entries that do
+     * not parse are ignored. */
+    blobs?: readonly (string | Readonly<Record<string, unknown>>)[] | string;
+    /** Files downloaded at once, 1–16. Default 4. */
+    concurrency?: number;
+    /** Largest file read, as in {@link MediaFetchOptions.maxBytes}. */
+    maxBytes?: number;
+    /** Ends every request and download. */
+    signal?: AbortSignal;
+};
+/** How resolved bytes are written into a part. */
+export interface MediaWriteOptions {
+    /**
+     * `"dataUrl"` (the default) writes text: a `data:` URL where the part's field takes one (OpenAI
+     * `image_url` and `file_data`, Responses `input_image`, `input_file` and `computer_screenshot`,
+     * AI SDK `image` and `file` `data`), and bare base64 where its convention requires it (GenAI
+     * `blob` `content`, OpenAI `input_audio` and assistant `audio` `data`, Anthropic `source.data`,
+     * Gemini `inlineData.data`, Vertex `media` `data`, Bedrock `source.bytes`), so the value can be
+     * sent to the provider again. `"bytes"` writes the `Uint8Array` itself in the same field.
+     */
+    as?: "dataUrl" | "bytes";
+}
+/** A part {@link MediaClient.resolve} or {@link rehydrateMedia} left unchanged. */
+export interface MediaResolveFailure {
+    /** RFC 6901 JSON pointer of the part in the value. */
+    pointer: string;
+    /** The SHA-256 the part records, or for `prefix_only` the SHA-256 of the entry naming its
+     * content field. */
+    sha256: string;
+    /** Why its bytes were not written: as {@link MediaFetchError.reason}, `not_found` when no local
+     * file has the digest, or `prefix_only` for a part whose content field holds only the first
+     * 16 KiB of its file, as a `blobs` entry names it; {@link MediaClient.fetch} returns that file
+     * whole. */
+    reason: MediaFetchErrorReason | "prefix_only";
+}
+/** A recorded value with the bytes of its digest parts written back. */
+export interface MediaResolution<T = unknown> {
+    /** A new value: the input is never changed. Each resolved part holds its bytes in its content
+     * field and no longer carries `sha256` and `size`. */
+    value: T;
+    /** Parts whose bytes were written. */
+    resolved: number;
+    /** Parts left unchanged, in reading order. */
+    failures: MediaResolveFailure[];
+}
+/** A local copy of a file {@link rehydrateMedia} reads by digest: a runner's verified case file
+ * (`LocalFile`, read from `path`) or bytes already in memory. */
+export type MediaSourceFile = {
+    /** Lowercase hex SHA-256 of the file's bytes. */
+    sha256: string;
+    /** The file's media type, used when the part declares none. */
+    contentType?: string;
+} & ({
+    /** Path of the file on this machine. */
+    path: string;
+    /** Not set when the file is given as a path. */
+    bytes?: undefined;
+} | {
+    /** The file's bytes. */
+    bytes: Uint8Array;
+    /** Not set when the file is given as bytes. */
+    path?: undefined;
+});
+/** The least time a download may go without a byte before it ends: two minutes. Tests shorten it;
+ * it is not exported from the package. */
+export declare const mediaDownload: {
+    minIdleMillis: number;
+};
+/**
+ * Hue's media reads, `client.media`: one recorded file by digest (`fetch`), and every digest part
+ * of a recorded value written back with its bytes (`resolve`). The project key is sent to Hue
+ * only; the file itself comes from a short-lived link fetched without it.
+ */
+export declare class MediaClient {
+    #private;
+    /**
+     * Downloads one recorded file and checks it against its digest: from a trace (Hue reads the
+     * span's record that lists the digest under `hue.blobs`) or from a snapshot's pinned copy. A
+     * read Hue answers as unavailable (503 or 429) is asked once more after its `Retry-After`, at
+     * most 10 seconds, and so is one that lost its connection or timed out.
+     *
+     * @throws MediaFetchError for a file Hue does not hold, a refused key, an unavailable store, a
+     * file over `maxBytes`, bytes that do not match the digest or a failed request.
+     * @throws TypeError for an invalid trace, span or snapshot ID or digest.
+     */
+    fetch(target: MediaFetchTarget, options?: MediaFetchOptions): Promise<MediaFile>;
+    /**
+     * Writes the bytes of every digest part in `value` back into its content field, the reverse of
+     * the SDK's media capture: each distinct digest is downloaded once with {@link fetch}, at most
+     * `concurrency` at a time. Returns a new value; a part whose file cannot be read stays as it is
+     * and is reported in `failures`. Parse attributes recorded as JSON text (such as
+     * `gen_ai.input.messages`) before passing them; pass the record's `hue.blobs` as `blobs` when
+     * reading a snapshot, whose copies carry no media type.
+     *
+     * @throws TypeError for invalid identifiers or options; never for one file's failure.
+     */
+    resolve<T>(value: T, options: MediaResolveOptions): Promise<MediaResolution<T>>;
+}
+/**
+ * Writes case files back into the digest parts of a case's inputs, by SHA-256: the files a runner
+ * hands a target (`context.files`) or any bytes the caller holds, so a target can send the
+ * recorded model call again with the real image, document or audio. Each file is read once and
+ * checked against its digest; a part with no matching file, or whose file does not match, stays
+ * as it is and is reported in `failures`. Returns a new value; the input is never changed.
+ */
+export declare function rehydrateMedia<T>(value: T, files: readonly MediaSourceFile[], options?: MediaWriteOptions): Promise<MediaResolution<T>>;