@hue-run/sdk 0.15.2 → 0.17.0

package/dist/evals/runner.js

@@ -213,6 +213,10 @@ async function allPages(page, maximum = 5000) {
  * world that cannot be sealed), telemetry not accepted by the one transport the cases share, or
  * a target outcome left uncertain because its world could not be sealed or read back. Starting
  * more cases into it only piles up the same failure; the run is resumed later instead. */
+/** A new attempt replacing an interrupted one, which Hue still holds as started. */
+function retry(previousExecutionId) {
+    return previousExecutionId ? { previousExecutionId, allowUncertainRetry: true } : {};
+}
 function systemic(error) {
     // A saved outcome still waiting for its trace (`TraceExportUnacknowledgedError`) is not
     // systemic: the flush failed on an earlier attempt, telemetry may have recovered since, and
@@ -230,6 +234,7 @@ async function pool(items, concurrency, execute) {
     let position = 0;
     const failures = [];
     let stop = false;
+    let cancelled;
     await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
         while (position < items.length && !stop) {
             const item = items[position++];
@@ -237,12 +242,20 @@ async function pool(items, concurrency, execute) {
                 await execute(item);
             }
             catch (error) {
+                // The caller's stop: start nothing more, and report the stop once the cases in flight settle.
+                if (error instanceof TargetCancelledError) {
+                    cancelled = error;
+                    stop = true;
+                    continue;
+                }
                 failures.push(error);
                 if (systemic(error))
                     stop = true;
             }
         }
     }));
+    if (cancelled)
+        throw cancelled;
     if (failures.length === 1)
         throw failures[0];
     if (failures.length) {
@@ -498,17 +511,19 @@ export async function runExperiment(options) {
                 scores,
                 exportState: "pending",
                 ...(saved.trace ? { trace: saved.trace } : {}),
+                ...(saved.attempt ? { attempt: saved.attempt } : {}),
             };
             await store.write(file, prepared);
             return prepared;
         }
         // A case pinned to a world keeps its files apart and removes them when it ends: the agent's
         // copies of its inputs, its work, the evaluator's downloads and the staged outputs.
-        const worldDirectoryOf = (itemId) => join(filesRoot, `world-case-${uuid(itemId)}`);
+        // A retry gets its own, so an agent a dead run left behind cannot write into it.
+        const worldDirectoryOf = (itemId, attempt) => join(filesRoot, `world-case-${uuid(itemId)}${attempt ? `-${uuid(attempt)}` : ""}`);
         const runCase = async (item, files) => {
             const file = `case-${uuid(item.id)}`;
-            const worldDirectory = worldDirectoryOf(item.id);
-            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? worldDirectory : join(filesRoot, `case-${uuid(item.id)}`);
+            const directDirectory = join(filesRoot, `case-${uuid(item.id)}`);
+            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? files.directory : directDirectory;
             let checkpoint;
             try {
                 checkpoint = await store.read(file);
@@ -521,6 +536,24 @@ export async function runExperiment(options) {
                 throw error;
             }
             files.keepOutputs = checkpoint?.stage === "uploading";
+            if (checkpoint && "attempt" in checkpoint)
+                files.directory = worldDirectoryOf(item.id, checkpoint.attempt);
+            let retryOf;
+            // A case left running by a process that died here is as interrupted as one the caller
+            // stopped; a fresh world isolates it from anything the dead run's agent still does.
+            if (checkpoint?.stage === "running" && store.reclaimed && options.rerunCrashedCases) {
+                checkpoint = { ...checkpoint, stage: "interrupted" };
+                await store.write(file, checkpoint);
+            }
+            if (checkpoint?.stage === "interrupted") {
+                retryOf = checkpoint.executionId;
+                checkpoint = undefined;
+                // A direct case's directory goes too, so the retry finds no input copy or output of the
+                // stopped attempt.
+                await removeCaseFiles(files.directory);
+                await removeCaseFiles(directDirectory);
+                files.directory = worldDirectoryOf(item.id, retryOf);
+            }
             if (checkpoint && checkpoint.stage !== "prepared" && checkpoint.stage !== "uploading") {
                 if (checkpoint.stage === "serialization_failed")
                     throw new OutcomeSerializationError(checkpoint.executionId);
@@ -528,6 +561,7 @@ export async function runExperiment(options) {
                     ? await options.client.startExecution(experiment.id, item.id, {
                         idempotencyKey: checkpoint.startKey,
                         traceExternalId: checkpoint.traceExternalId,
+                        ...retry(checkpoint.previousExecutionId),
                     })
                     : await options.client.getExecution(checkpoint.executionId);
                 throw new UncertainExecutionError(item.id, execution.id);
@@ -542,7 +576,9 @@ export async function runExperiment(options) {
                 files.keepOutputs = false;
             }
             if (!checkpoint) {
-                if (item.execution)
+                if (options.signal?.aborted)
+                    throw new TargetCancelledError();
+                if (item.execution && item.execution.id !== retryOf)
                     throw new UncertainExecutionError(item.id, item.execution.id);
                 const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
                 if (frozenCase.datasetVersionId !== version.id)
@@ -574,13 +610,16 @@ export async function runExperiment(options) {
                         stage: "starting",
                         startKey: randomUUID(),
                         traceExternalId: span.traceId,
+                        ...(retryOf ? { previousExecutionId: retryOf } : {}),
                     };
                     await store.write(file, start);
                     const execution = await options.client.startExecution(experiment.id, item.id, {
                         idempotencyKey: start.startKey,
                         traceExternalId: span.traceId,
+                        ...retry(retryOf),
                     });
-                    await store.write(file, { stage: "running", executionId: execution.id });
+                    const attempt = retryOf ? { attempt: retryOf } : {};
+                    await store.write(file, { stage: "running", executionId: execution.id, ...attempt });
                     let state = "succeeded";
                     let output;
                     let generated;
@@ -604,6 +643,17 @@ export async function runExperiment(options) {
                     catch (error) {
                         if (error instanceof TargetOutcomeUncertainError)
                             throw error;
+                        // A stop the caller asked for is no outcome: the execution stays open for the
+                        // resumed run to replace instead of being scored as cancelled.
+                        if (error instanceof TargetCancelledError && options.signal?.aborted) {
+                            options.hue.recordError(span.span, error);
+                            await store.write(file, {
+                                stage: "interrupted",
+                                executionId: execution.id,
+                                ...attempt,
+                            });
+                            throw error;
+                        }
                         state = error instanceof TargetCancelledError ? "cancelled" : "error";
                         targetError = error;
                         options.hue.recordError(span.span, error);
@@ -659,6 +709,7 @@ export async function runExperiment(options) {
                         ...(options.persistResultContent && output !== undefined ? { output } : {}),
                         ...(state === "error" ? { error: errorPayload(targetError) } : {}),
                         files: staged,
+                        ...attempt,
                     };
                     // Without persisted result content a restart cannot reconstruct the outcome; the
                     // saved stage then reports the execution as uncertain instead of guessing.
@@ -838,21 +889,23 @@ export async function runExperiment(options) {
             // A world case's directory goes however the case ends, a forced exit included. An
             // interrupted upload keeps only the staged outputs it resumes from; inputs are downloaded
             // anew for scoring. A direct case has no such directory, so nothing is removed for it.
-            const worldDirectory = worldDirectoryOf(item.id);
-            const files = { keepOutputs: true };
-            const untrack = onForcedExit(() => releaseWorldFilesSync(worldDirectory, files.keepOutputs));
+            const files = { keepOutputs: true, directory: worldDirectoryOf(item.id) };
+            const untrack = onForcedExit(() => releaseWorldFilesSync(files.directory, files.keepOutputs));
             try {
                 await runCase(item, files);
-                await releaseWorldFiles(worldDirectory, false);
+                await releaseWorldFiles(files.directory, false);
             }
             catch (error) {
-                await releaseWorldFiles(worldDirectory, files.keepOutputs);
+                await releaseWorldFiles(files.directory, files.keepOutputs);
                 throw error;
             }
             finally {
                 untrack();
             }
         });
+        // A stop that came while the last cases saved their outcomes still leaves the run open.
+        if (options.signal?.aborted)
+            throw new TargetCancelledError();
         let finish = await store.read("finish");
         if (!finish) {
             finish = { key: randomUUID() };

package/dist/evals/scorer-publication.js

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { json, digest } from "./json.js";
 const metricName = z
     .string()

package/dist/evals/simulation.js

@@ -391,7 +391,7 @@ export async function runSimulation(options) {
             attempt.stage !== "completed" &&
             attempt.scenarioDigest !== scenarioDigest &&
             attempt.scenarioDigest !== previousDigest)
-            throw new Error("Recover the unfinished simulation before running a changed scenario or agent revision");
+            throw new Error("Recover the unfinished simulation before running a changed scenario or agent revision: run the unchanged one again to finish it, or use another checkpoint directory");
         if (!attempt || attempt.stage === "completed") {
             attempt = {
                 scenarioDigest,
@@ -417,12 +417,14 @@ export async function runSimulation(options) {
             ];
         }
         const experimentId = checkpointSegment(attempt.experimentId, "experiment id");
-        const runUrl = new URL(`/experiments/${experimentId}`, options.client.baseUrl).toString();
+        const runUrl = new URL(`/runs/${experimentId}`, options.client.baseUrl).toString();
         await options.onProgress?.({ type: "run_created", experimentId, runUrl });
         const requested = requestedConfiguration
             ? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(experimentId)).config)
             : undefined;
         const report = await runExperiment({
+            signal: options.signal,
+            rerunCrashedCases: true,
             client: options.client,
             hue: options.hue,
             experimentId,

package/dist/inline-files.d.ts

@@ -1,6 +1,6 @@
-/** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
+/** Text media above this decoded byte limit is offloaded; binary media always uses a digest. */
 export declare const INLINE_FILE_LIMIT: number;
-/** The longest message text, in UTF-8 bytes, inspected for inline files. */
+/** The longest message text, in UTF-16 code units, inspected for inline files. */
 export declare const MAX_INLINE_FILE_TEXT: number;
 /** Message text one record inspects for inline files in all, in UTF-16 code units. */
 export declare const INLINE_FILE_TEXT_PER_RECORD: number;
@@ -20,38 +20,47 @@ export interface InlineFileBytes {
  * `data:` URL whose data does not decode is taken as UTF-8 too.
  */
 export declare function inlineFileBytes(content: string): InlineFileBytes;
-/** The key holding a part's inline content: GenAI `blob` parts and AI SDK 6 `file` parts. */
-export declare function contentKey(part: Record<string, unknown>): "content" | "data" | undefined;
-/**
- * Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
- * with their SHA-256 and byte size, so a span that inlines a large file exports the file's
- * identity instead of being rejected for its size. The part keeps its other fields (`type`,
- * `mime_type`, `modality`, `mediaType`, …). Other attributes, shorter messages and values that
- * are not JSON are returned unchanged.
- */
-export declare function hashInlineFiles(key: string, value: unknown, 
-/** Counts the files replaced by their digest, when given. */
-count?: {
-    files: number;
-}): unknown;
-/** One inline file part found in a parsed message: the part object, the key holding its inline
- * content, that content, and the part's content as a JSON Pointer (RFC 6901) into the message. */
+/** A part and the exact leaf that contains its bytes. Nested provider metadata stays in place. */
 export interface InlineFilePart {
     part: Record<string, unknown>;
-    key: "content" | "data";
+    owner: Record<string, unknown>;
+    key: string;
     content: string;
     pointer: string;
+    partPointer: string;
+    contentType?: string;
+    name?: string;
+    base64?: boolean;
 }
-/**
- * The inline file parts of a parsed message whose content text is longer than
- * {@link INLINE_FILE_LIMIT} UTF-8 bytes, in document order: the caller decodes each and keeps the
- * ones whose file bytes are over the limit. The parts are the parsed value's own objects, so the
- * caller may replace their content in place.
- */
+export declare function isMediaName(value: unknown): value is string;
+/** Whether the receiver can preview these bytes as text. */
+export declare function isTextMediaType(contentType: string): boolean;
+/** Cheap shared prefilter before bounded parsing of span message attributes. */
+export declare function mayInlineFiles(key: string, value: unknown): boolean;
+/** A printable MIME type with optional parameters, accepted by the upload contract. */
+export declare function isMediaContentType(value: unknown): value is string;
+/** Detect the content pointer, MIME, name, and encoding contract of a provider media part. */
+export declare function contentPath(part: Record<string, unknown>, pointer?: string): InlineFilePart | undefined;
+/** Inline media in document order. The caller applies the decoded text threshold. */
 export declare function largeInlineFileParts(value: unknown): InlineFilePart[];
-/** The digest an inline file part is exported as when its bytes are not uploaded: the part's
- * other fields with the file's SHA-256 and byte size, as {@link hashInlineFiles} writes it. */
-export declare function digestPart(part: Record<string, unknown>, key: "content" | "data", digest: {
+export declare function mediaBytes(file: InlineFilePart): InlineFileBytes;
+export declare function mediaContentType(file: InlineFilePart, decoded: InlineFileBytes): string;
+export interface FileDigest {
     sha256: string;
     size: number;
-}): void;
+}
+/** Bounded payload identity memo, shared by admission and export, cleared at shutdown. */
+export declare class MediaDigestMemo {
+    private entries;
+    private units;
+    clear(): void;
+    digest(file: InlineFilePart, decoded?: InlineFileBytes, traceId?: string): FileDigest;
+    bytes(bytes: Uint8Array, traceId: string): FileDigest;
+}
+/** Digest placement removes only the leaf and adds identity to the provider part. */
+export declare function digestPart(file: InlineFilePart, digest: FileDigest): void;
+export declare function hashInlineFiles(key: string, value: unknown, count?: {
+    files: number;
+}, memo?: MediaDigestMemo, traceId?: string): unknown;
+/** Nested message attributes retain small media and digest larger media without upload state. */
+export declare function hashNestedInlineFiles(key: string, value: unknown): unknown;