@hue-run/sdk 0.10.0 → 0.11.0

package/dist/client.js

@@ -8,12 +8,13 @@ import { LoggerProvider } from "@opentelemetry/sdk-logs";
 import { TracerProvider } from "@opentelemetry/sdk-trace";
 import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
 import { HUE_SCOPE } from "./config.js";
-import { hostedServerAddresses, hostedToolActivity, hostedToolProvider } from "./provider-tools.js";
+import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
+import { toolCatalogSummary } from "./tool-definitions.js";
 import { encodeContent, noopSpan, safeSpan } from "./safety.js";
 import { createHueTransport, HueExportError } from "./transport.js";
 import { verifyTrace } from "./receipt.js";
 import { sdkVersion } from "./version.js";
-import { MAX_FILE_DATA_BYTES } from "./config.js";
+import { MAX_BODY_BYTES, MAX_FILE_DATA_BYTES } from "./config.js";
 /**
  * Thrown by {@link HueClient.checkConnection} and {@link HueClient.verifyTrace} when Hue cannot be
  * reached, rejects the project key or answers unexpectedly. The message is fixed and safe to log;
@@ -149,6 +150,9 @@ class ContextualTracer {
     }
 }
 const propagator = new W3CTraceContextPropagator();
+/** A provider tool listing's bounds before its catalog summary: one export request, 64 levels and
+ * 65,536 values, as the Python SDK's content snapshot. */
+const catalogLimits = { bytes: MAX_BODY_BYTES, nodes: 65_536, depth: 64 };
 /**
  * Hue tracing client. Helpers create spans through a private tracer and local async context, never
  * through global OpenTelemetry registration, and they fail open: capture problems are counted in
@@ -517,11 +521,14 @@ export class HueClient {
      * `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
      * `code_interpreter_call` items, and Anthropic Messages `mcp_tool_use` / `server_tool_use`
      * blocks with their result blocks. Each becomes an `execute_tool {name}` child span of the active
-     * (or given) context with `gen_ai.tool.type` `extension` and `gen_ai.tool.call.id`; MCP calls add
+     * (or given) context with `gen_ai.tool.type` `extension`, `gen_ai.tool.call.id` and
+     * `hue.tool.call.position` (the item's 0-based position in the response); MCP calls add
      * `mcp.server.name` (the provider's label, or the `servers` entry for it). Arguments and results
-     * follow `captureContent`; a failed call carries `error.type` and ERROR status. An OpenAI
-     * `mcp_list_tools` item becomes a `tools/list` child span carrying that server's tools as
-     * `gen_ai.tool.definitions`. Call it inside `hue.model()` so the spans nest under the model call
+     * follow `captureContent`; a failed call carries `error.type` and ERROR status, and with content
+     * capture a failed OpenAI MCP call's status description is the provider's scrubbed, bounded error
+     * text. An OpenAI `mcp_list_tools` item becomes a `tools/list` child span carrying that server's
+     * tools as `gen_ai.tool.definitions`, or without content capture only `hue.tool.names` and
+     * `hue.tool.definitions.sha256`. Call it inside `hue.model()` so the spans nest under the model call
      * and `provider` defaults to its provider; pass `request` to record each server's host as
      * `server.address`. The spans have no duration of their own: the provider ran the tools inside
      * the model request. Unreadable items are skipped and counted; nothing is thrown.
@@ -564,9 +571,12 @@ export class HueClient {
                     attributes["server.address"] = address;
                 return attributes;
             };
-            const fail = (span, type) => {
+            const fail = (span, type, description) => {
                 span.setAttribute("error.type", type);
-                span.setStatus({ code: SpanStatusCode.ERROR });
+                span.setStatus({
+                    code: SpanStatusCode.ERROR,
+                    ...(description === undefined ? {} : { message: description }),
+                });
             };
             for (const call of activity.calls) {
                 const span = this.tracer.startSpan(`execute_tool ${call.name}`, {
@@ -576,6 +586,7 @@ export class HueClient {
                         "gen_ai.tool.name": call.name,
                         "gen_ai.tool.type": "extension",
                         ...(call.callId === undefined ? {} : { "gen_ai.tool.call.id": call.callId }),
+                        "hue.tool.call.position": call.position,
                         ...server(call.server),
                     },
                 }, parent);
@@ -584,7 +595,11 @@ export class HueClient {
                 if (call.result !== undefined)
                     this.setContent(span, "gen_ai.tool.call.result", call.result);
                 if (call.errorType !== undefined)
-                    fail(span, call.errorType);
+                    fail(span, call.errorType, 
+                    // The provider's error text is content: exported only when content is captured.
+                    this.captureContent && call.errorText !== undefined
+                        ? providerErrorDescription(call.errorText)
+                        : undefined);
                 span.end();
             }
             for (const listing of activity.listings) {
@@ -592,7 +607,18 @@ export class HueClient {
                     kind: SpanKind.INTERNAL,
                     attributes: { "mcp.method.name": "tools/list", ...server(listing.server) },
                 }, parent);
-                this.setContent(span, "gen_ai.tool.definitions", listing.definitions);
+                if (this.captureContent)
+                    this.setContent(span, "gen_ai.tool.definitions", listing.definitions);
+                // Names and the catalog digest are metadata: without content, the listing carries the
+                // summary metadata-only export gives any record's tool definitions, within the summary's
+                // bounds rather than one content field's.
+                else
+                    try {
+                        span.setAttributes(toolCatalogSummary(encodeContent(listing.definitions, catalogLimits)));
+                    }
+                    catch {
+                        this.transport.instrumentationFailure();
+                    }
                 if (listing.errorType !== undefined)
                     fail(span, listing.errorType);
                 span.end();

package/dist/evals/client.d.ts

@@ -14,9 +14,13 @@ export interface EvaluationClientOptions {
 export declare class HueApiError extends Error {
     /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
     readonly status?: number | undefined;
+    /** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
+    readonly retryAfterSeconds?: number | undefined;
     constructor(
     /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
-    status?: number | undefined);
+    status?: number | undefined, 
+    /** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
+    retryAfterSeconds?: number | undefined);
 }
 /** An artifact download that ran past the size its caller expected; the rest is not read. */
 export declare class ArtifactSizeError extends Error {

package/dist/evals/client.js

@@ -4,11 +4,15 @@ import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptR
 /** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
 export class HueApiError extends Error {
     status;
+    retryAfterSeconds;
     constructor(
     /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
-    status) {
+    status, 
+    /** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
+    retryAfterSeconds) {
         super(status ? `Hue API request failed (HTTP ${status})` : "Hue API connection or response failed");
         this.status = status;
+        this.retryAfterSeconds = retryAfterSeconds;
         this.name = "HueApiError";
     }
 }
@@ -106,19 +110,21 @@ function productRunFields(value, kind) {
 const REFUSAL_RETRIES = 4;
 /** A refusal asking for a longer wait fails at once rather than holding the caller. */
 const MAX_REFUSAL_RETRY_AFTER_SECONDS = 5;
+/** The whole seconds a 429 or 503 asks the caller to wait, up to a day, when it says. */
+function askedRetryAfter(response) {
+    if (response.status !== 429 && response.status !== 503)
+        return undefined;
+    const header = response.headers.get("retry-after")?.trim() ?? "";
+    return /^\d{1,6}$/.test(header) ? Math.min(Number(header), 86_400) : undefined;
+}
 /**
  * The whole-second `Retry-After` of a 429 or 503 asking for at most 5 seconds. Hue sends one only
  * when it refused the request before acting on it, such as a busy key check, so sending any method
  * again is safe. A date, a longer wait or any other failure, a timeout included, is not retried.
  */
 function refusalRetryAfter(response) {
-    if (response.status !== 429 && response.status !== 503)
-        return undefined;
-    const header = response.headers.get("retry-after")?.trim() ?? "";
-    if (!/^\d{1,6}$/.test(header))
-        return undefined;
-    const seconds = Number(header);
-    return seconds <= MAX_REFUSAL_RETRY_AFTER_SECONDS ? seconds : undefined;
+    const seconds = askedRetryAfter(response);
+    return seconds !== undefined && seconds <= MAX_REFUSAL_RETRY_AFTER_SECONDS ? seconds : undefined;
 }
 /**
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
@@ -178,7 +184,7 @@ export class EvaluationClient {
         }));
         if (!response.ok) {
             await response.body?.cancel();
-            throw new HueApiError(response.status);
+            throw new HueApiError(response.status, askedRetryAfter(response));
         }
         try {
             const reader = response.body?.getReader();
@@ -220,7 +226,7 @@ export class EvaluationClient {
         }));
         if (!response.ok) {
             await response.body?.cancel();
-            throw new HueApiError(response.status);
+            throw new HueApiError(response.status, askedRetryAfter(response));
         }
         try {
             const reader = response.body?.getReader();
@@ -273,7 +279,7 @@ export class EvaluationClient {
         }
         await response.body?.cancel().catch(() => undefined);
         if (!response.ok)
-            throw new HueApiError(response.status);
+            throw new HueApiError(response.status, askedRetryAfter(response));
     }
     page(options = {}) {
         const query = new URLSearchParams();

package/dist/evals/files.d.ts

@@ -99,10 +99,18 @@ export declare function downloadCaseInputs(client: EvaluationClient, files: Case
 }>;
 /** Copy the target's generated files next to the checkpoint and record their identities. */
 export declare function stageOutputFiles(files: OutputFile[], directory: string): Promise<StagedOutputFile[]>;
+/** How long an upload waits for Hue to verify one artifact, and how often it looks. Hue verifies
+ * within a two-minute lease, which can outlast one request's timeout. */
+export declare const artifactSettling: {
+    settleMillis: number;
+    pollMillis: number;
+    maxPollMillis: number;
+};
 /**
  * Publish staged files as verified artifacts. Reservations use stable keys derived from the
- * execution and the bytes, so a resumed upload settles on the same artifact.
+ * execution and the bytes, so a resumed upload settles on the same artifact, including one Hue
+ * finished verifying after an earlier attempt stopped waiting.
  */
-export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>): Promise<void>;
+export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>, timing?: typeof artifactSettling): Promise<void>;
 /** Project uploaded staged files as the verified local files handed to scorers. */
 export declare function localOutputFiles(files: StagedOutputFile[]): LocalFile[];

package/dist/evals/files.js

@@ -321,11 +321,74 @@ export async function stageOutputFiles(files, directory) {
     }
     return staged;
 }
+/** How long an upload waits for Hue to verify one artifact, and how often it looks. Hue verifies
+ * within a two-minute lease, which can outlast one request's timeout. */
+export const artifactSettling = { settleMillis: 180_000, pollMillis: 1000, maxPollMillis: 10_000 };
+/** A completion failure that can mean Hue is still verifying: a lost or timed-out response, a
+ * verification already running (409), or a refusal to retry (429, 503). */
+function mayStillVerify(error) {
+    return (error instanceof HueApiError &&
+        (error.status === undefined ||
+            error.status === 409 ||
+            error.status === 429 ||
+            error.status === 503));
+}
+/**
+ * Completes an artifact's verification and returns the state it settles in. When a completion
+ * times out, finds verification already running or is refused for now, the artifact is read after
+ * a pause: ready is returned; verifying is completed again, which Hue refuses while its lease
+ * holds and restarts once it lapsed; an artifact a retryable refusal (a lost response, 429 or 503)
+ * left unverified is completed again up to three times; any other state is returned for the
+ * caller to refuse (a resume uploads a rejected artifact again). All within `timing.settleMillis`.
+ */
+async function settleArtifact(client, id, timing) {
+    const deadline = Date.now() + timing.settleMillis;
+    let pause = timing.pollMillis;
+    let retries = 0;
+    for (;;) {
+        let refusedForNow;
+        let asked = 0;
+        try {
+            return (await client.completeArtifact(id)).state;
+        }
+        catch (error) {
+            if (!mayStillVerify(error) || Date.now() >= deadline)
+                throw error;
+            refusedForNow = error.status !== 409;
+            // A refusal that says how long to wait is waited out, within the settling window.
+            asked = (error.retryAfterSeconds ?? 0) * 1000;
+        }
+        await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(Math.max(pause, asked), deadline - Date.now()))));
+        pause = Math.min(pause * 2, timing.maxPollMillis);
+        let current;
+        try {
+            current = await client.getArtifact(id);
+        }
+        catch (error) {
+            if (!mayStillVerify(error))
+                throw error;
+        }
+        if (current?.state === "ready")
+            return current.state;
+        const unverified = current &&
+            ["reserved", "rejected"].includes(current.state) &&
+            current.failureCode !== "mismatch";
+        if (current && current.state !== "verifying" && !(unverified && refusedForNow && retries++ < 3))
+            return current.state;
+        if (Date.now() >= deadline) {
+            const seconds = Math.round(timing.settleMillis / 1000);
+            throw new Error(current?.state === "verifying"
+                ? `Hue was still verifying generated file ${id} after ${seconds} seconds`
+                : `Hue had not verified generated file ${id} after ${seconds} seconds (${current ? `it was ${current.state}` : "its state could not be read"})`);
+        }
+    }
+}
 /**
  * Publish staged files as verified artifacts. Reservations use stable keys derived from the
- * execution and the bytes, so a resumed upload settles on the same artifact.
+ * execution and the bytes, so a resumed upload settles on the same artifact, including one Hue
+ * finished verifying after an earlier attempt stopped waiting.
  */
-export async function uploadOutputFiles(client, executionId, files, save) {
+export async function uploadOutputFiles(client, executionId, files, save, timing = artifactSettling) {
     for (const file of files) {
         if (file.artifactId)
             continue;
@@ -339,7 +402,8 @@ export async function uploadOutputFiles(client, executionId, files, save) {
         });
         let state = reserved.state;
         if (state !== "ready") {
-            if (reserved.copyState !== "acknowledged") {
+            // A verification already running holds the uploaded bytes; Hue refuses another upload then.
+            if (reserved.copyState !== "acknowledged" && state !== "verifying") {
                 const upload = await client.requestArtifactUpload(reserved.id);
                 try {
                     await client.uploadArtifactBytes(upload, bytes, file.contentType);
@@ -349,18 +413,7 @@ export async function uploadOutputFiles(client, executionId, files, save) {
                     // settled by verified completion; never rewrite a final object here.
                 }
             }
-            let attempt = 0;
-            for (;;) {
-                try {
-                    state = (await client.completeArtifact(reserved.id)).state;
-                    break;
-                }
-                catch (error) {
-                    if (!(error instanceof HueApiError) || error.status !== 503 || attempt++ >= 2)
-                        throw error;
-                    await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
-                }
-            }
+            state = await settleArtifact(client, reserved.id, timing);
         }
         if (state !== "ready")
             throw new Error(`Generated file ${file.filename} was not verified by Hue (${state})`);

package/dist/evals/json.js

@@ -4,27 +4,96 @@ export function aggregateBounds(bytes) {
     return { bytes, nodes: Math.ceil(bytes / 2), depth: valueBounds.depth + 8 };
 }
 const isText = (item) => item.isWellFormed() && !item.includes("\u0000");
+/** A string's code points: its UTF-16 length less one for each surrogate pair. */
+function codePoints(text) {
+    let count = text.length;
+    for (let index = 0; index < text.length - 1; index++) {
+        const unit = text.charCodeAt(index);
+        if (unit < 0xd800 || unit > 0xdbff)
+            continue;
+        const next = text.charCodeAt(index + 1);
+        if (next >= 0xdc00 && next <= 0xdfff) {
+            count--;
+            index++;
+        }
+    }
+    return count;
+}
+/** Thrown within `json` when the byte bound is passed, which is checked last. */
+class PastByteBound extends Error {
+}
 /** Reject lossy JSON serialization before creating requests/checkpoints. */
 export function json(value, requested = valueBounds) {
     const bounds = typeof requested === "number" ? { ...valueBounds, bytes: requested } : requested;
     const ancestors = new Set();
     let nodes = 0;
+    // The UTF-8 length of the JSON text, counted as the value is read, so a value too long to
+    // serialize (past the runtime's longest string) is refused before `JSON.stringify` is asked to.
+    let bytes = 0;
+    // Once the byte bound is passed, the value is read again only to check it (below).
+    let checking = false;
+    // Text the second read has found valid is not checked again, so a string referenced many times
+    // is scanned once: by content up to 16,383 UTF-16 units, and a longer string when it is the last
+    // checked of its length. V8 hashes those by length alone, so a set of them would compare each
+    // with every other of that length.
+    const checkedText = new Set();
+    const checkedLong = new Map();
+    const text = (item) => {
+        if (!checking)
+            return isText(item);
+        if (item.length <= 16_383 ? checkedText.has(item) : checkedLong.get(item.length) === item)
+            return true;
+        if (!isText(item))
+            return false;
+        if (item.length <= 16_383)
+            checkedText.add(item);
+        else
+            checkedLong.set(item.length, item);
+        return true;
+    };
+    const charge = (amount) => {
+        if (checking)
+            return;
+        bytes += amount;
+        if (bytes > bounds.bytes)
+            throw new PastByteBound();
+    };
+    // A string's JSON text is at least as long as the string, so one longer than what is left is
+    // refused before it is escaped.
+    const chargeText = (text) => {
+        if (checking)
+            return;
+        if (text.length + 2 > bounds.bytes - bytes)
+            throw new PastByteBound();
+        charge(Buffer.byteLength(JSON.stringify(text)));
+    };
     const visit = (item, depth) => {
         if (++nodes > bounds.nodes || depth > bounds.depth)
             throw new RangeError("JSON exceeds depth/node limits");
-        if (item === null || typeof item === "boolean")
+        if (item === null || typeof item === "boolean") {
+            charge(item === false ? 5 : 4);
             return item;
-        if (typeof item === "number" && Number.isFinite(item))
+        }
+        if (typeof item === "number" && Number.isFinite(item)) {
+            charge(JSON.stringify(item).length);
             return item;
-        if (typeof item === "string" && isText(item))
+        }
+        if (typeof item === "string" && text(item)) {
+            chargeText(item);
             return item;
+        }
         if (!item || typeof item !== "object" || ancestors.has(item))
             throw new TypeError("Expected finite JSON without cycles or invalid Unicode");
         ancestors.add(item);
         try {
             if (Array.isArray(item)) {
+                // Each element is a value: an array longer than the values left is refused before its
+                // keys are listed.
+                if (item.length > bounds.nodes - nodes)
+                    throw new RangeError("JSON exceeds depth/node limits");
                 if (Object.keys(item).length !== item.length)
                     throw new TypeError("Sparse/extended arrays are not JSON");
+                charge(item.length ? item.length + 1 : 2);
                 return item.map((entry) => visit(entry, depth + 1));
             }
             if (Object.getPrototypeOf(item) !== Object.prototype && Object.getPrototypeOf(item) !== null)
@@ -32,9 +101,33 @@ export function json(value, requested = valueBounds) {
             if (Object.getOwnPropertySymbols(item).length)
                 throw new TypeError("JSON cannot contain symbol properties");
             const result = Object.create(null);
-            for (const key of Object.keys(item).sort()) {
-                if (!isText(key))
+            // Each member is a value: an object with more than the values left is refused before its
+            // keys are sorted.
+            const keys = Object.keys(item);
+            if (keys.length > bounds.nodes - nodes)
+                throw new RangeError("JSON exceeds depth/node limits");
+            if (!checking) {
+                // Keys whose JSON text alone (at least their code points, two quotes and a colon each)
+                // needs more bytes than are left pass the byte bound before they are sorted, as in the
+                // Python SDK, so both read such an object the same way. A key has at least half its
+                // UTF-16 length in code points, so a long one passes it without counting them.
+                let left = bounds.bytes - bytes;
+                for (const key of keys) {
+                    left -= 3;
+                    if (key.length / 2 > left)
+                        throw new PastByteBound();
+                    left -= codePoints(key);
+                    if (left < 0)
+                        throw new PastByteBound();
+                }
+                keys.sort();
+            }
+            charge(keys.length ? keys.length + 1 : 2);
+            for (const key of keys) {
+                if (!text(key))
                     throw new TypeError("Expected finite JSON without cycles or invalid Unicode");
+                chargeText(key);
+                charge(1);
                 const descriptor = Object.getOwnPropertyDescriptor(item, key);
                 if (!("value" in descriptor))
                     throw new TypeError("JSON cannot contain accessors");
@@ -47,10 +140,24 @@ export function json(value, requested = valueBounds) {
             ancestors.delete(item);
         }
     };
-    const result = visit(value, 0);
-    if (Buffer.byteLength(JSON.stringify(result)) > bounds.bytes)
-        throw new RangeError("JSON exceeds byte limit");
-    return result;
+    try {
+        const result = visit(value, 0);
+        if (Buffer.byteLength(JSON.stringify(result)) > bounds.bytes)
+            throw new RangeError("JSON exceeds byte limit");
+        return result;
+    }
+    catch (error) {
+        if (!(error instanceof PastByteBound))
+            throw error;
+    }
+    // The byte bound is checked last, as it was before the length was counted as the value is
+    // read: a value past it that holds a value that is not JSON, or passes the value or depth bound,
+    // is refused for that. So the value is read again with each object's keys in their own order,
+    // and nothing is counted, escaped or sorted, nor text checked twice (`text`).
+    checking = true;
+    nodes = 0;
+    visit(value, 0);
+    throw new RangeError("JSON exceeds byte limit");
 }
 export function digest(value) {
     return createHash("sha256")

package/dist/evals/runner.js

@@ -9,7 +9,7 @@ import { loadEnvironmentEvidence } from "./environment-evidence.js";
 import { CheckpointStore } from "./checkpoint.js";
 import { onForcedExit } from "./exit-cleanup.js";
 import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
-import { json, uuid } from "./json.js";
+import { json, uuid, valueBounds } from "./json.js";
 import { executableHere, persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
 import { TargetResult, } from "./types.js";
 /**
@@ -41,6 +41,23 @@ export class OutcomeSerializationError extends Error {
         this.name = "OutcomeSerializationError";
     }
 }
+/** The bound `json` names when a valid output is over what Hue stores for one case. */
+const outputLimits = new Map([
+    ["JSON exceeds byte limit", "bytes"],
+    ["JSON exceeds depth/node limits", "structure"],
+]);
+/** `200000` as `200,000`, without depending on the runtime's locale data. */
+const grouped = (value) => String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+/** A target's output beyond what Hue stores for one case. Recorded as that case's error (type
+ * `OutputTooLarge`), never thrown: the other cases keep running. */
+class OutputTooLargeError extends Error {
+    constructor(limit) {
+        super(limit === "bytes"
+            ? `The output is larger than ${grouped(valueBounds.bytes)} bytes of JSON, the most Hue stores for one case; return a large result as a generated file`
+            : `The output has more than ${grouped(valueBounds.nodes)} JSON values or nests deeper than ${valueBounds.depth} levels, the most Hue stores for one case`);
+        this.name = "OutputTooLargeError";
+    }
+}
 /** Thrown when cooperative caller cancellation stops target execution. */
 export class TargetCancelledError extends Error {
     constructor() {
@@ -313,7 +330,7 @@ export async function runExperiment(options) {
         const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
         const sanitize = (message) => message.slice(0, 4000).toWellFormed().replaceAll("\u0000", "");
         const errorPayload = (error) => ({
-            type: "TargetError",
+            type: error instanceof OutputTooLargeError ? "OutputTooLarge" : "TargetError",
             ...(options.persistResultContent && error instanceof Error
                 ? { message: sanitize(error.message) }
                 : {}),
@@ -476,12 +493,22 @@ export async function runExperiment(options) {
                         try {
                             output = json(output);
                         }
-                        catch {
-                            await store.write(file, {
-                                stage: "serialization_failed",
-                                executionId: execution.id,
-                            });
-                            throw new OutcomeSerializationError(execution.id);
+                        catch (error) {
+                            // An output over the size, count or depth bounds is the target's own failure, as
+                            // a file it cannot deliver is: the case completes as an error and the run goes
+                            // on. Output that is not JSON at all still stops the run for inspection.
+                            const limit = error instanceof RangeError ? outputLimits.get(error.message) : undefined;
+                            if (limit === undefined) {
+                                await store.write(file, {
+                                    stage: "serialization_failed",
+                                    executionId: execution.id,
+                                });
+                                throw new OutcomeSerializationError(execution.id);
+                            }
+                            state = "error";
+                            output = undefined;
+                            targetError = new OutputTooLargeError(limit);
+                            options.hue.recordError(span.span, targetError);
                         }
                     }
                     if (output !== undefined)

package/dist/evals/scorer-publication.js

@@ -30,6 +30,47 @@ const worldOutcomeMetrics = [
     "process_constraints",
     "task_success",
 ].map((name) => ({ name, type: "boolean" }));
+/** `hue.conversion_outcome.v2`: v1's rubric with the destination reported as `recipient`,
+ * `thread` and `subject`. */
+const conversionOutcomeV2Metrics = [
+    "completed_run",
+    "saved_draft",
+    "correct_destination",
+    "recipient",
+    "thread",
+    "subject",
+    "content",
+    "unrelated_preserved",
+    "process_constraints",
+    "task_success",
+].map((name) => ({ name, type: "boolean" }));
+/** `hue.outcome_assertions.v2`'s counts; `.v3` adds the judge counts. */
+const outcomeAssertionMetrics = [
+    { name: "task_success", type: "boolean" },
+    ...["assertions_passed", "assertions_failed", "advisory_failed", "agent_mistakes"].map((name) => ({ name, type: "number", min: 0 })),
+];
+const outcomeAssertionV3Metrics = [
+    ...outcomeAssertionMetrics,
+    ...["judges_passed", "judges_failed", "judges_advisory"].map((name) => ({
+        name,
+        type: "number",
+        min: 0,
+    })),
+];
+/** The judge pin a `hue.outcome_assertions.v3` version carries; every field is required. */
+const outcomeJudgeConfig = z.strictObject({
+    model: z.string().min(1).max(200),
+    provider: z.string().min(1).max(64),
+    template: z.string().regex(/^[a-f0-9]{64}$/),
+    samples: z.number().int().min(1).max(9),
+    temperature: z.number().min(0).max(2),
+    maxOutputTokens: z.number().int().min(1).max(32_000),
+    timeoutMs: z.number().int().min(1_000).max(300_000),
+});
+/** A Hue-executed entry whose metrics are fixed by the entry itself. */
+const fixedMetrics = (fixed) => metrics
+    .default(fixed)
+    .refine((value) => digest(value) === digest(fixed), "World outcome metrics are fixed by the pinned entry");
 const jsonValue = z.custom((value) => {
     try {
         json(value);
@@ -85,9 +126,23 @@ const sdkScorerPublication = z.union([
     z.strictObject({
         kind: z.literal("world_outcome"),
         entry: z.literal("hue.conversion_outcome.v1"),
-        metrics: metrics
-            .default(worldOutcomeMetrics)
-            .refine((value) => digest(value) === digest(worldOutcomeMetrics), "World outcome metrics are fixed by the pinned entry"),
+        metrics: fixedMetrics(worldOutcomeMetrics),
+    }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.conversion_outcome.v2"),
+        metrics: fixedMetrics(conversionOutcomeV2Metrics),
+    }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.outcome_assertions.v2"),
+        metrics: fixedMetrics(outcomeAssertionMetrics),
+    }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.outcome_assertions.v3"),
+        metrics: fixedMetrics(outcomeAssertionV3Metrics),
+        config: z.strictObject({ judge: outcomeJudgeConfig }),
     }),
     z.strictObject({ kind: z.literal("manual"), metrics }),
     z.strictObject({ kind: z.literal("llm_judge"), config: judgeConfig, metrics }),