agent-hitch 0.2.7 → 0.2.9

package/dist/src/evals/planned-retry-execution.js

@@ -0,0 +1,174 @@
+import path from "node:path";
+import { runInfrastructureRetries } from "./infrastructure-retry.js";
+import { runRemoteInfrastructureRetries } from "./remote-infrastructure-retry.js";
+import { beginPlannedInfrastructureRetry } from "./planned-retry-lifecycle.js";
+import { localSourceBackendFailure } from "./result-helpers.js";
+import { resourceRequirementForTask } from "./execution-plan-resources.js";
+import { resolvedImageMapping } from "./environment-image-planning.js";
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+import { workSchedulingPriority } from "./planned-execution-support.js";
+import { ensurePhysicalRetryDecision, ensureTerminalRetryDecision, readEvalRetryState } from "./retry-state.js";
+import { classifyTrialFailure, physicalRetryAllowed } from "./failure-classifier.js";
+import { physicalRetryWorkItem } from "./physical-retry-work.js";
+import { retryBackoffMs } from "./retry-backoff.js";
+export async function runPlannedInfrastructureRetries(options, publisher, semaphore, metrics, workItem, completed) {
+    if (options.signal?.aborted || completed.run.backend.process_exit_code !== 0 || completed.run.rawResult === null)
+        return [];
+    if (options.localTransport && localSourceBackendFailure(completed.run.rawResult))
+        return [];
+    for (const trigger of completed.refs.filter((ref) => ref.observation_status === "invalid")) {
+        const classification = classifyTrialFailure(trigger);
+        if (physicalRetryAllowed(classification) && options.request.infrastructure_retries > 0)
+            continue;
+        const decision = await ensureTerminalRetryDecision({
+            evalDirectory: options.evalDirectory, evalId: options.evalId, item: workItem, retryIndex: 1, trigger,
+            exhausted: physicalRetryAllowed(classification),
+        });
+        if (decision)
+            options.sink.emit({
+                type: "eval.retry.skipped", work_id: workItem.work_id, decision_id: decision.decision_id,
+                task_id: trigger.task_id, attempt: trigger.attempt, disposition: decision.disposition,
+                classification: decision.classification, state: decision.state,
+            });
+        if (decision && providerCircuitFailure(decision.classification.code))
+            options.sink.emit({
+                type: "eval.provider-circuit.opened",
+                scope: "trial-retry",
+                mode: "retry-only",
+                provider: workItem.provider,
+                model: options.request.model || null,
+                work_id: workItem.work_id,
+                decision_id: decision.decision_id,
+                stable_code: decision.classification.code,
+                automatic_probe: false,
+            });
+    }
+    return runPlannedInfrastructureRetriesFromSource(options, publisher, semaphore, metrics, workItem, completed, 1);
+}
+function providerCircuitFailure(code) {
+    return code === "provider_quota_exhausted"
+        || code === "provider_auth_failed"
+        || code === "provider_configuration_invalid";
+}
+export async function runPlannedInfrastructureRetriesFromSource(options, publisher, semaphore, metrics, workItem, source, firstRetryIndex) {
+    const progress = publisher.current();
+    if (options.remoteWorkExecutor) {
+        return (await runRemoteInfrastructureRetries({
+            options, item: workItem, initial: source, progress, firstRetryIndex, metrics,
+            replaceProgressTrial: (ref, retryWorkId) => publisher.replaceInvalid(ref, retryWorkId),
+            currentProgress: () => publisher.current(),
+        })).runs;
+    }
+    const retries = await runInfrastructureRetries({
+        evalId: options.evalId,
+        evalDirectory: options.evalDirectory,
+        backendBaseDirectory: path.join(options.evalDirectory, "harbor", "work-items", source.workId, "infrastructure-retries"),
+        logicalAttempt: source.attempt,
+        initialRefs: source.refs,
+        progress,
+        request: options.request,
+        root: options.root,
+        resolvedRevision: options.resolvedRevision,
+        controllerRuntime: options.controllerRuntime,
+        preparedArtifact: preparedArtifactForWorkItem(options, workItem),
+        executionResources: resourceRequirementForTask(options.plan, source.tasks[0])?.main_limits ?? options.plan.default_trial_resources,
+        resolvedImages: resolvedImageMapping(workItem.image_refs ?? []),
+        ...(source.environmentImages ? { environmentImages: source.environmentImages } : {}),
+        beginRetry: async ({ retry, triggers, backendDirectory }) => {
+            const directRelease = await semaphore?.acquire(options.signal, workSchedulingPriority(workItem));
+            try {
+                const lifecycle = await beginPlannedInfrastructureRetry({
+                    options, item: workItem, retry, triggers, backendDirectory,
+                    ...(source.environmentImages ? { environmentImages: source.environmentImages } : {}),
+                });
+                return {
+                    ...lifecycle,
+                    close: async () => { try {
+                        await lifecycle.close();
+                    }
+                    finally {
+                        directRelease?.();
+                    } },
+                };
+            }
+            catch (error) {
+                directRelease?.();
+                throw error;
+            }
+        },
+        replaceProgressTrial: (ref, retryWorkId) => publisher.replaceInvalid(ref, retryWorkId),
+        currentProgress: () => publisher.current(),
+        originWorkItem: workItem,
+        firstRetryIndex,
+        onRetryExecutionStarted: (workId) => metrics.startWork(workId, "retry"),
+        onRetryExecutionFinished: (workId) => metrics.finishWork(workId),
+        onRetryBackoff: (durationMs) => metrics.addBlocked("backoff", durationMs),
+        onVerifierDuration: (durationMs) => metrics.addVerifier(durationMs),
+        ...(options.onWorkItemQueued ? { onRetryWorkPlanned: options.onWorkItemQueued } : {}),
+        ...(options.localTransport ? { localTransport: options.localTransport } : {}),
+        ...(options.plan.model_capture ? { modelCapturePlan: options.plan.model_capture } : {}),
+        ...(options.interactionCaptureExporter ? { interactionCaptureExporter: options.interactionCaptureExporter } : {}),
+        env: options.env,
+        ...(options.harborExecutable !== undefined ? { harborExecutable: options.harborExecutable } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(options.trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs: options.trialBundleGraceMs }),
+        sink: options.sink,
+        ...(options.localTransport ? { stopAfterResult: localSourceBackendFailure } : {}),
+    });
+    return retries.runs;
+}
+export async function reconcilePersistedInitialRetryDecisions(options) {
+    let state = await readEvalRetryState(options.evalDirectory, options.evalId);
+    for (const trigger of options.progress.trials.filter((trial) => trial.observation_status === "invalid")) {
+        if (state?.decisions.some((decision) => decision.trigger_trial_id === trigger.trial_id))
+            continue;
+        const item = options.plan.work_items.find((entry) => entry.logical_attempt === trigger.attempt && entry.task_ids.includes(trigger.task_id));
+        if (!item)
+            throw new TypeError(`persisted invalid trial is absent from execution plan: ${trigger.task_id}#${trigger.attempt}`);
+        const classification = classifyTrialFailure(trigger);
+        if (physicalRetryAllowed(classification) && options.request.infrastructure_retries > 0) {
+            const retryWork = physicalRetryWorkItem(item, 1, [trigger]);
+            await ensurePhysicalRetryDecision({
+                evalDirectory: options.evalDirectory, evalId: options.evalId, item, retryIndex: 1, trigger,
+                notBefore: new Date(Date.now() + retryBackoffMs(options.request.infrastructure_retry_backoff_ms, 1, retryWork.work_id)).toISOString(),
+            });
+        }
+        else {
+            await ensureTerminalRetryDecision({
+                evalDirectory: options.evalDirectory, evalId: options.evalId, item, retryIndex: 1, trigger,
+                exhausted: physicalRetryAllowed(classification),
+            });
+        }
+        state = await readEvalRetryState(options.evalDirectory, options.evalId);
+    }
+}
+export async function pendingRetryWork(options) {
+    const state = await readEvalRetryState(options.evalDirectory, options.evalId);
+    const pending = new Map();
+    if (!state)
+        return pending;
+    const seen = new Set();
+    for (const decision of state.decisions.filter((entry) => entry.disposition === "physical-retry" && entry.state === "planned")) {
+        if (!decision.retry_work_id || seen.has(decision.retry_work_id))
+            continue;
+        seen.add(decision.retry_work_id);
+        const item = options.plan.work_items.find((entry) => entry.slots.includes(decision.slot_id));
+        if (!item || item.logical_attempt === null || item.task_ids.length !== 1)
+            throw new TypeError(`retry decision slot is absent from execution plan: ${decision.slot_id}`);
+        const persistedTrigger = options.progress.trials.find((trial) => trial.trial_id === decision.trigger_trial_id);
+        const trigger = persistedTrigger ?? (decision.trigger_run_id ? {
+            trial_id: decision.trigger_trial_id, run_id: decision.trigger_run_id, task_id: item.task_ids[0],
+            attempt: item.logical_attempt, observation_status: "invalid", invalid_reason: decision.classification.code,
+        } : undefined);
+        if (!trigger)
+            throw new TypeError(`retry decision trigger cannot be recovered: ${decision.decision_id}`);
+        const taskId = item.task_ids[0];
+        const entries = pending.get(taskId) ?? [];
+        entries.push({ item, decision, trigger });
+        pending.set(taskId, entries);
+    }
+    for (const entries of pending.values())
+        entries.sort((left, right) => left.decision.retry_index - right.decision.retry_index);
+    return pending;
+}
+//# sourceMappingURL=planned-retry-execution.js.map

package/dist/src/evals/planned-retry-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-retry-execution.js","sourceRoot":"","sources":["../../../src/evals/planned-retry-execution.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAErE,OAAO,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,+BAA+B,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AAEvE,OAAO,EAAE,sBAAsB,EAA0B,MAAM,gCAAgC,CAAC;AAKhG,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEhH,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAQpD,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,OAAoC,EACpC,SAA4B,EAC5B,SAAwC,EACxC,OAA6B,EAC7B,QAA2B,EAC3B,SAA4B;IAE5B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5H,IAAI,OAAO,CAAC,cAAc,IAAI,yBAAyB,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5F,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QAC3F,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,oBAAoB,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC;YAAE,SAAS;QACjG,MAAM,QAAQ,GAAG,MAAM,2BAA2B,CAAC;YACjD,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO;YACpG,SAAS,EAAE,oBAAoB,CAAC,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,IAAI,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC9B,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACxF,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACrF,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK;aAC/D,CAAC,CAAC;QACH,IAAI,QAAQ,IAAI,sBAAsB,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtF,IAAI,EAAE,8BAA8B;gBACpC,KAAK,EAAE,aAAa;gBACpB,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI;gBACpC,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,WAAW,EAAE,QAAQ,CAAC,cAAc,CAAC,IAAI;gBACzC,eAAe,EAAE,KAAK;aACvB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,yCAAyC,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;AACnH,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI,KAAK,0BAA0B;WACrC,IAAI,KAAK,sBAAsB;WAC/B,IAAI,KAAK,gCAAgC,CAAC;AACjD,CAAC;AAUD,MAAM,CAAC,KAAK,UAAU,yCAAyC,CAC7D,OAAoC,EACpC,SAA4B,EAC5B,SAAwC,EACxC,OAA6B,EAC7B,QAA2B,EAC3B,MAAmB,EACnB,eAAuB;IAEvB,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;IACrC,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,OAAO,CAAC,MAAM,8BAA8B,CAAC;YAC3C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO;YAC5E,oBAAoB,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC;YACtF,eAAe,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE;SAC3C,CAAC,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,wBAAwB,CAAC;QAC7C,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,wBAAwB,CAAC;QACvH,cAAc,EAAE,MAAM,CAAC,OAAO;QAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;QACxB,QAAQ;QACR,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,gBAAgB,EAAE,2BAA2B,CAAC,OAAO,EAAE,QAAQ,CAAC;QAChE,kBAAkB,EAAE,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAW,CAAC,EAAE,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,uBAAuB;QAC5I,cAAc,EAAE,oBAAoB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,UAAU,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAC1D,MAAM,aAAa,GAAG,MAAM,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjG,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,+BAA+B,CAAC;oBACtD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB;oBAC1D,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACrF,CAAC,CAAC;gBACH,OAAO;oBACL,GAAG,SAAS;oBACZ,KAAK,EAAE,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC;wBAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;oBAAC,CAAC;4BAAS,CAAC;wBAAC,aAAa,EAAE,EAAE,CAAC;oBAAC,CAAC,CAAC,CAAC;iBACvF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE,CAAC;gBAClB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC;QACtF,eAAe,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE;QAC1C,cAAc,EAAE,QAAQ;QACxB,eAAe;QACf,uBAAuB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC;QACvE,wBAAwB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;QAChE,cAAc,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC;QACzE,kBAAkB,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC;QACnE,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7E,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,0BAA0B,EAAE,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACvG,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,yBAAyB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClF,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,IAAI,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uCAAuC,CAAC,OAAoC;IAChG,IAAI,KAAK,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QACxG,IAAI,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,KAAK,OAAO,CAAC,QAAQ,CAAC;YAAE,SAAS;QAClG,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,KAAK,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5I,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,0DAA0D,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/H,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,oBAAoB,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC,EAAE,CAAC;YACvF,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5D,MAAM,2BAA2B,CAAC;gBAChC,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO;gBAC1F,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE;aACtI,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,2BAA2B,CAAC;gBAChC,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO;gBAC1F,SAAS,EAAE,oBAAoB,CAAC,cAAc,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;QACD,KAAK,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAoC;IACzE,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,IAAI,CAAC,KAAK;QAAE,OAAO,OAAO,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,gBAAgB,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;QAC9H,IAAI,CAAC,QAAQ,CAAC,aAAa,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,SAAS;QAC1E,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,sDAAsD,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACxK,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QAC/G,MAAM,OAAO,GAAG,gBAAgB,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;YAC7D,QAAQ,EAAE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,EAAE,QAAQ,CAAC,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW;YACzG,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,kBAAkB,EAAE,SAAkB,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,IAAI;SACpH,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACf,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzG,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;QAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9H,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/evals/planned-retry-lifecycle.js

@@ -5,8 +5,10 @@ import { resourceRequirementForTask, runtimeResourcesForTask } from "./execution
 import { recordLocalDockerProcessExit, recordLocalDockerProcessStart, releaseLocalDockerProcessRecord } from "./local-docker-provider.js";
 import { resolvedImageMapping } from "./environment-image-planning.js";
 import { prebuiltTaskImage, verifyTrialEnvironmentImageExecution } from "./trial-environment-evidence.js";
+import { physicalRetryWorkItem } from "./physical-retry-work.js";
 export async function beginPlannedInfrastructureRetry(input) {
-    const { options, item } = input;
+    const { options } = input;
+    const item = physicalRetryWorkItem(input.item, input.retry, input.triggers);
     const taskId = item.task_ids[0];
     let permit;
     let lease;
@@ -52,7 +54,7 @@ export async function beginPlannedInfrastructureRetry(input) {
             attempt: item.logical_attempt,
             trigger_trials: input.triggers.map((trial) => trial.trial_id).sort(),
         });
-        return retryLifecycle(input, lease, permit, runningAnnounced);
+        return retryLifecycle({ ...input, item }, lease, permit, runningAnnounced);
     }
     catch (error) {
         if (lease)

package/dist/src/evals/planned-retry-lifecycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"planned-retry-lifecycle.js","sourceRoot":"","sources":["../../../src/evals/planned-retry-lifecycle.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAEnI,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAEpG,OAAO,EAAE,4BAA4B,EAAE,6BAA6B,EAAE,+BAA+B,EAAE,MAAM,4BAA4B,CAAC;AAE1I,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,oCAAoC,EAAE,MAAM,iCAAiC,CAAC;AAG1G,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,KAOrD;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,IAAI,MAAyF,CAAC;IAC9F,IAAI,KAAuC,CAAC;IAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe;YAC5C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,8BAA8B;gBACpC,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,aAAa,EAAE,MAAM,CAAC,YAAY;gBAClC,cAAc,EAAE,MAAM,CAAC,aAAa;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACH,KAAK,GAAG,MAAM,oBAAoB,CAAC;YACjC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YAChG,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,8BAA8B;SACtC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;QACpC,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACxE,gBAAgB,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,6BAA6B;YACnC,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,IAAI,CAAC,eAAe;YAC7B,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;SACrE,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7E,IAAI,gBAAgB;YAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7H,MAAM,EAAE,OAAO,EAAE,CAAC;QAClB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,KAMC,EACD,KAA2B,EAC3B,MAAyG,EACzG,gBAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IACpC,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,2BAA2B,CAAC;QAC3C,SAAS;QACT,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;QACjC,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB;QACnD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,gBAAyB,CAAC;IAC9B,IAAI,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,gBAAgB,KAAK,SAAS;YAAE,OAAO;QAC3C,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,eAAe;gBACrB,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;gBACvB,WAAW,EAAE,OAAO,CAAC,KAAK;gBAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,UAAU,EAAE,OAAO,CAAC,UAAU;aAC/B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,MAAiC,CAAC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,GAAkB,EAAE,CAAC,MAAM,KAAK,mBAAmB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACrP,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,wBAAwB,EAAE,KAAK,IAAI,EAAE;YACnC,IAAI,gBAAgB,KAAK,SAAS;gBAAE,MAAM,gBAAgB,CAAC;YAC3D,OAAO,oCAAoC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,EAAE;YACP,kBAAkB,EAAE,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,uBAAuB;YACzH,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,eAAe,EAAE,SAAS;YAC1B,cAAc,EAAE,oBAAoB,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAC3D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC;gBAC/E,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,6BAA6B,CAAC;oBAC/D,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;oBACjC,aAAa,EAAE,OAAO,CAAC,aAAa;oBACpC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;oBACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;oBACxC,GAAG;iBACJ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC1C,eAAe,EAAE,CAAC,MAA8D,EAAE,EAAE,CAAC,4BAA4B,CAAC;oBAChH,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;oBACL,GAAG,MAAM;iBACV,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;aACzB,CAAC,CAAC,CAAC,EAAE,CAAC;SACR;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAUlC;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IAC1C,IAAI,OAAgB,CAAC;IACrB,MAAM,OAAO,GAAG,KAAK,EAAE,SAAiC,EAAiB,EAAE;QACzE,IAAI,CAAC;YAAC,MAAM,SAAS,EAAE,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAK,CAAC;QAAC,CAAC;IACjE,CAAC,CAAC;IACF,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,gBAAgB,EAAE,CAAC;IACrC,IAAI,KAAK,CAAC,eAAe,EAAE;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,+BAA+B,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/I,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,oCAAoC;YAC1C,cAAc,EAAE,+BAA+B;YAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO;YAC7B,WAAW,EAAE,QAAQ,CAAC,KAAK;YAC3B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;YACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7H,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,sCAAsC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7O,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mCAAmC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAG,KAA2B,CAAC,IAAI,IAAI,sBAAsB,EAAE,CAAC,CAAC;QACtN,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,gBAAgB;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/I,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IACxB,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC"}
+{"version":3,"file":"planned-retry-lifecycle.js","sourceRoot":"","sources":["../../../src/evals/planned-retry-lifecycle.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAEnI,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAEpG,OAAO,EAAE,4BAA4B,EAAE,6BAA6B,EAAE,+BAA+B,EAAE,MAAM,4BAA4B,CAAC;AAE1I,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,oCAAoC,EAAE,MAAM,iCAAiC,CAAC;AAE1G,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,KAOrD;IACC,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC1B,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,IAAI,MAAyF,CAAC;IAC9F,IAAI,KAAuC,CAAC;IAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe;YAC5C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,8BAA8B;gBACpC,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,aAAa,EAAE,MAAM,CAAC,YAAY;gBAClC,cAAc,EAAE,MAAM,CAAC,aAAa;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACH,KAAK,GAAG,MAAM,oBAAoB,CAAC;YACjC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YAChG,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,8BAA8B;SACtC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;QACpC,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACxE,gBAAgB,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,6BAA6B;YACnC,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,IAAI,CAAC,eAAe;YAC7B,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;SACrE,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7E,IAAI,gBAAgB;YAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7H,MAAM,EAAE,OAAO,EAAE,CAAC;QAClB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,KAMC,EACD,KAA2B,EAC3B,MAAyG,EACzG,gBAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IACpC,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,2BAA2B,CAAC;QAC3C,SAAS;QACT,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;QACjC,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB;QACnD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,gBAAyB,CAAC;IAC9B,IAAI,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,gBAAgB,KAAK,SAAS;YAAE,OAAO;QAC3C,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,eAAe;gBACrB,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;gBACvB,WAAW,EAAE,OAAO,CAAC,KAAK;gBAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,UAAU,EAAE,OAAO,CAAC,UAAU;aAC/B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,MAAiC,CAAC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,GAAkB,EAAE,CAAC,MAAM,KAAK,mBAAmB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACrP,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,wBAAwB,EAAE,KAAK,IAAI,EAAE;YACnC,IAAI,gBAAgB,KAAK,SAAS;gBAAE,MAAM,gBAAgB,CAAC;YAC3D,OAAO,oCAAoC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,EAAE;YACP,kBAAkB,EAAE,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,uBAAuB;YACzH,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,eAAe,EAAE,SAAS;YAC1B,cAAc,EAAE,oBAAoB,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAC3D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC;gBAC/E,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,6BAA6B,CAAC;oBAC/D,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;oBACjC,aAAa,EAAE,OAAO,CAAC,aAAa;oBACpC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;oBACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;oBACxC,GAAG;iBACJ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC1C,eAAe,EAAE,CAAC,MAA8D,EAAE,EAAE,CAAC,4BAA4B,CAAC;oBAChH,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;oBACL,GAAG,MAAM;iBACV,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;aACzB,CAAC,CAAC,CAAC,EAAE,CAAC;SACR;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAUlC;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IAC1C,IAAI,OAAgB,CAAC;IACrB,MAAM,OAAO,GAAG,KAAK,EAAE,SAAiC,EAAiB,EAAE;QACzE,IAAI,CAAC;YAAC,MAAM,SAAS,EAAE,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAK,CAAC;QAAC,CAAC;IACjE,CAAC,CAAC;IACF,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,gBAAgB,EAAE,CAAC;IACrC,IAAI,KAAK,CAAC,eAAe,EAAE;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,+BAA+B,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/I,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,oCAAoC;YAC1C,cAAc,EAAE,+BAA+B;YAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO;YAC7B,WAAW,EAAE,QAAQ,CAAC,KAAK;YAC3B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;YACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7H,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,sCAAsC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7O,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mCAAmC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAG,KAA2B,CAAC,IAAI,IAAI,sBAAsB,EAAE,CAAC,CAAC;QACtN,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,gBAAgB;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/I,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IACxB,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC"}

package/dist/src/evals/preparation-rerun.js

@@ -0,0 +1,225 @@
+import { cp, lstat, readdir, rename } from "node:fs/promises";
+import path from "node:path";
+import { HitchError, SCHEMA_VERSION, atomicWriteJSON, ensureDir, readJSON } from "../foundation/index.js";
+import { logicalPlanModelCapture, readEvalLogicalPlan } from "./eval-logical-plan.js";
+import { readEvalProgress } from "./progress.js";
+import { resolveLocalDatasetTaskIds, validateEvalId } from "./request.js";
+import { evalRerunSemantics } from "./rerun-types.js";
+import { invalidTrialSlots, slotKey, sortSlots, uniqueTasks, validateProgressPlan } from "./rerun-slots.js";
+import { runEval } from "./service.js";
+/** Restart an eval that failed before its executable plan was fully materialized. */
+export async function restartIncompleteEval(options, request, startedAt, rerunDirectory, statePath) {
+    const materializedFiles = ["plan.json", "execution-plan.json", "progress.json"];
+    if ((await Promise.all(materializedFiles.map((name) => exists(path.join(options.evalDirectory, name))))).every(Boolean))
+        return null;
+    const failedResult = await readJSON(path.join(options.evalDirectory, "result.json"), null);
+    if (failedResult === null)
+        return null;
+    if (options.rerunType !== "candidate-restart" || options.selector.mode !== "invalid") {
+        throw unavailable("an eval with incomplete preparation can only be restarted with candidate-restart --invalid");
+    }
+    if (failedResult?.status === "cancelled") {
+        throw new HitchError("cancelled eval cannot be rerun", { code: "eval_rerun_cancelled", exitCode: 2 });
+    }
+    if (failedResult?.status !== "failed" || !Array.isArray(failedResult.trials) || failedResult.trials.length !== 0) {
+        throw unavailable("eval preparation is incomplete but the source is not a failed pre-execution attempt");
+    }
+    const taskIds = await resolveLocalDatasetTaskIds(request.dataset);
+    if (taskIds === null || taskIds.length === 0) {
+        throw unavailable("incomplete preparation recovery requires an enumerable local task plan");
+    }
+    const logicalPlan = await readEvalLogicalPlan(options.evalDirectory, options.evalId, request).catch((error) => {
+        throw unavailable(error.message);
+    });
+    let modelCapturePlan = logicalPlanModelCapture(logicalPlan) ?? options.modelCapturePlan;
+    if (!modelCapturePlan && !await exists(path.join(options.evalDirectory, "submission.json"))) {
+        modelCapturePlan = defaultModelCapturePlan();
+    }
+    if (!modelCapturePlan) {
+        throw unavailable("incomplete preparation recovery requires the original model capture plan");
+    }
+    await writeRequest(rerunDirectory, options, startedAt, [], []);
+    await writeState(statePath, options, startedAt, "running", [], [], [], []);
+    await archiveIncompleteAttempt(options, rerunDirectory, failedResult);
+    try {
+        const restarted = await runEval({
+            evalId: validateEvalId(options.evalId),
+            root: options.root,
+            request,
+            normalizedRequest: request,
+            precreated: true,
+            replaceTerminal: true,
+            env: options.env ?? process.env,
+            ...(options.maxConcurrentOverride === undefined ? {} : { maxConcurrentOverride: options.maxConcurrentOverride }),
+            ...(options.harborExecutable === undefined ? {} : { harborExecutable: options.harborExecutable }),
+            ...(options.signal ? { signal: options.signal } : {}),
+            ...(options.trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs: options.trialBundleGraceMs }),
+            ...(options.executionResources ? { executionResources: options.executionResources } : {}),
+            ...(options.executionResourceSource ? { executionResourceSource: options.executionResourceSource } : {}),
+            executionStrategy: options.executionStrategy
+                ?? (logicalPlan?.attempt_execution === "harbor-task-slots-v1" ? "local-task-slots-v1" : "legacy-attempt-shards"),
+            ...(options.environmentBuildMode ? { environmentBuildMode: options.environmentBuildMode } : {}),
+            modelCapturePlan,
+            ...(options.harborArtifactBuilder ? { harborArtifactBuilder: options.harborArtifactBuilder } : {}),
+        });
+        if (restarted.status === "cancelled") {
+            throw new HitchError("eval rerun was aborted", { code: "eval_rerun_aborted", exitCode: 9 });
+        }
+        const [planValue, executionPlanValue, progress] = await Promise.all([
+            readJSON(path.join(options.evalDirectory, "plan.json"), null),
+            readJSON(path.join(options.evalDirectory, "execution-plan.json"), null),
+            readEvalProgress(options.evalDirectory),
+        ]);
+        if (planValue === null || executionPlanValue === null || progress === null)
+            throw restartFailure(restarted);
+        const plan = parseRecoveredPlan(planValue, options.evalId, request);
+        validateProgressPlan(progress, plan, request, options.evalId);
+        const selectedTrials = sortSlots(plan.tasks.flatMap((taskId) => Array.from({ length: plan.attempts }, (_, index) => ({ task_id: taskId, attempt: index + 1 }))));
+        const valid = new Set(progress.trials.filter((trial) => trial.observation_status === "valid").map(slotKey));
+        const repairedTrials = selectedTrials.filter((trial) => valid.has(slotKey(trial)));
+        const remainingTrials = invalidTrialSlots(plan.tasks, plan.attempts, progress);
+        const selectedTasks = uniqueTasks(selectedTrials);
+        const repairedTasks = uniqueTasks(repairedTrials);
+        const remainingTasks = uniqueTasks(remainingTrials);
+        const completedAt = new Date().toISOString();
+        await writeRequest(rerunDirectory, options, startedAt, selectedTasks, selectedTrials);
+        const output = {
+            schema_version: "1",
+            kind: "eval-rerun",
+            rerun_id: options.rerunId,
+            rerun_type: options.rerunType,
+            semantics: evalRerunSemantics(options.rerunType),
+            eval_id: options.evalId,
+            status: "completed",
+            selected_tasks: selectedTasks,
+            repaired_tasks: repairedTasks,
+            remaining_invalid_tasks: remainingTasks,
+            selected_trials: selectedTrials,
+            repaired_trials: repairedTrials,
+            remaining_invalid_trials: remainingTrials,
+            eval_status: restarted.status === "succeeded" ? "succeeded" : "failed",
+            started_at: startedAt,
+            completed_at: completedAt,
+        };
+        await writeState(statePath, options, startedAt, "completed", selectedTasks, selectedTrials, repairedTasks, repairedTrials, {
+            completedAt,
+            evalStatus: output.eval_status,
+            remainingTasks,
+            remainingTrials,
+        });
+        return output;
+    }
+    catch (error) {
+        await writeState(statePath, options, startedAt, "failed", [], [], [], [], {
+            completedAt: new Date().toISOString(),
+            errorCode: error instanceof HitchError ? error.code : "eval_rerun_failed",
+        }).catch(() => { });
+        throw error;
+    }
+}
+async function archiveIncompleteAttempt(options, rerunDirectory, result) {
+    const archive = path.join(rerunDirectory, "previous-attempt");
+    await ensureDir(archive);
+    for (const name of ["request.json", "submission.json", "control.json", "result.json"]) {
+        const source = path.join(options.evalDirectory, name);
+        if (await exists(source))
+            await cp(source, path.join(archive, name), { recursive: true, errorOnExist: true });
+    }
+    const stable = new Set(["request.json", "submission.json", "control.json", "result.json", "reruns"]);
+    for (const entry of await readdir(options.evalDirectory, { withFileTypes: true })) {
+        if (stable.has(entry.name))
+            continue;
+        await rename(path.join(options.evalDirectory, entry.name), path.join(archive, entry.name));
+    }
+    const failure = result.error && typeof result.error === "object" && !Array.isArray(result.error)
+        ? result.error
+        : null;
+    await atomicWriteJSON(path.join(archive, "recovery.json"), {
+        schema_version: SCHEMA_VERSION,
+        kind: "eval-preparation-restart",
+        eval_id: options.evalId,
+        rerun_id: options.rerunId,
+        ...(typeof result.failure_stage === "string" ? { failure_stage: result.failure_stage } : {}),
+        ...(typeof failure?.code === "string" ? { failure_code: failure.code } : {}),
+        archived_at: new Date().toISOString(),
+    });
+}
+function restartFailure(result) {
+    const error = result.error && typeof result.error === "object" && !Array.isArray(result.error)
+        ? result.error
+        : null;
+    const code = typeof error?.code === "string" && error.code ? error.code : "eval_rerun_preparation_failed";
+    const stage = typeof result.failure_stage === "string" ? result.failure_stage : "preparing";
+    const message = typeof error?.message === "string" && error.message ? error.message : "eval restart did not produce an executable plan";
+    return new HitchError(`eval restart failed during ${stage}: ${message}`, {
+        code,
+        exitCode: Number.isSafeInteger(result.exit_code) ? result.exit_code : 12,
+    });
+}
+function parseRecoveredPlan(value, evalId, request) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw unavailable("recovered eval plan is missing");
+    const plan = value;
+    if (plan.schema_version !== SCHEMA_VERSION || plan.eval_id !== evalId || plan.dataset !== request.dataset
+        || plan.benchmark_id !== request.benchmark_id || plan.benchmark_revision !== request.benchmark_revision
+        || !Number.isSafeInteger(plan.attempts) || plan.attempts !== request.attempts) {
+        throw unavailable("recovered eval plan identity changed");
+    }
+    if (!Array.isArray(plan.tasks) || plan.tasks.length === 0
+        || plan.tasks.some((task) => typeof task !== "string" || task.length === 0)
+        || new Set(plan.tasks).size !== plan.tasks.length)
+        throw unavailable("recovered eval task plan is invalid");
+    return { tasks: [...plan.tasks], attempts: plan.attempts };
+}
+async function writeRequest(directory, options, startedAt, tasks, trials) {
+    await atomicWriteJSON(path.join(directory, "request.json"), {
+        schema_version: SCHEMA_VERSION,
+        rerun_id: options.rerunId,
+        eval_id: options.evalId,
+        rerun_type: options.rerunType,
+        semantics: evalRerunSemantics(options.rerunType),
+        mode: options.selector.mode,
+        tasks: [...tasks],
+        trials: sortSlots(trials),
+        base_generation: 0,
+        created_at: startedAt,
+    });
+}
+async function writeState(file, options, startedAt, status, tasks, trials, repairedTasks, repairedTrials, terminal = {}) {
+    await atomicWriteJSON(file, {
+        schema_version: SCHEMA_VERSION,
+        rerun_id: options.rerunId,
+        eval_id: options.evalId,
+        rerun_type: options.rerunType,
+        semantics: evalRerunSemantics(options.rerunType),
+        status,
+        tasks: [...tasks],
+        trials: sortSlots(trials),
+        repaired_tasks: [...repairedTasks],
+        repaired_trials: sortSlots(repairedTrials),
+        started_at: startedAt,
+        ...(terminal.completedAt ? { completed_at: terminal.completedAt } : {}),
+        ...(terminal.evalStatus ? { eval_status: terminal.evalStatus } : {}),
+        ...(terminal.remainingTasks ? { remaining_invalid_tasks: [...terminal.remainingTasks] } : {}),
+        ...(terminal.remainingTrials ? { remaining_invalid_trials: sortSlots(terminal.remainingTrials) } : {}),
+        ...(terminal.errorCode ? { error: { code: terminal.errorCode } } : {}),
+    });
+}
+async function exists(file) {
+    try {
+        await lstat(file);
+        return true;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return false;
+        throw error;
+    }
+}
+function unavailable(message) {
+    return new HitchError(message, { code: "eval_rerun_unavailable", exitCode: 2 });
+}
+function defaultModelCapturePlan() {
+    return { requested_mode: "native", effective_mode: "native", required: false };
+}
+//# sourceMappingURL=preparation-rerun.js.map

package/dist/src/evals/preparation-rerun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"preparation-rerun.js","sourceRoot":"","sources":["../../../src/evals/preparation-rerun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC1G,OAAO,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AACtF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,0BAA0B,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE5G,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AASvC,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAwB,EACxB,OAAoB,EACpB,SAAiB,EACjB,cAAsB,EACtB,SAAiB;IAEjB,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,qBAAqB,EAAE,eAAe,CAAC,CAAC;IAChF,IAAI,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACrI,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3H,IAAI,YAAY,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,OAAO,CAAC,SAAS,KAAK,mBAAmB,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,WAAW,CAAC,4FAA4F,CAAC,CAAC;IAClH,CAAC;IACD,IAAI,YAAY,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;QACzC,MAAM,IAAI,UAAU,CAAC,gCAAgC,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,YAAY,EAAE,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,qFAAqF,CAAC,CAAC;IAC3G,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,0BAA0B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClE,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,WAAW,CAAC,wEAAwE,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC5G,MAAM,WAAW,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IACH,IAAI,gBAAgB,GAAG,uBAAuB,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,gBAAgB,CAAC;IACxF,IAAI,CAAC,gBAAgB,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC5F,gBAAgB,GAAG,uBAAuB,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,MAAM,WAAW,CAAC,0EAA0E,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,YAAY,CAAC,cAAc,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/D,MAAM,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3E,MAAM,wBAAwB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;IAEtE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;YAC9B,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;YACtC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,iBAAiB,EAAE,OAAO;YAC1B,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,IAAI;YACrB,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;YAC/B,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAChH,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;YACvG,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,GAAG,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE,uBAAuB,EAAE,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;mBACvC,CAAC,WAAW,EAAE,iBAAiB,KAAK,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,uBAAuB,CAAC;YAClH,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,gBAAgB;YAChB,GAAG,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnG,CAAC,CAAC;QACH,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,UAAU,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9F,CAAC;QACD,MAAM,CAAC,SAAS,EAAE,kBAAkB,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClE,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC;YAC7E,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,qBAAqB,CAAC,EAAE,IAAI,CAAC;YACvF,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC;SACxC,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,IAAI,kBAAkB,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI;YAAE,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;QAE5G,MAAM,IAAI,GAAG,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACpE,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CACxE,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,EACzB,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CACxD,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5G,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnF,MAAM,eAAe,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/E,MAAM,aAAa,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,YAAY,CAAC,cAAc,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;QACtF,MAAM,MAAM,GAAoB;YAC9B,cAAc,EAAE,GAAG;YACnB,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,OAAO,CAAC,OAAO;YACzB,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,SAAS,EAAE,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC;YAChD,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,MAAM,EAAE,WAAW;YACnB,cAAc,EAAE,aAAa;YAC7B,cAAc,EAAE,aAAa;YAC7B,uBAAuB,EAAE,cAAc;YACvC,eAAe,EAAE,cAAc;YAC/B,eAAe,EAAE,cAAc;YAC/B,wBAAwB,EAAE,eAAe;YACzC,WAAW,EAAE,SAAS,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YACtE,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,WAAW;SAC1B,CAAC;QACF,MAAM,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,EAAE;YACzH,WAAW;YACX,UAAU,EAAE,MAAM,CAAC,WAAW;YAC9B,cAAc;YACd,eAAe;SAChB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;YACxE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB;SAC1E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAwB,EACxB,cAAsB,EACtB,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IAC9D,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,aAAa,CAAC,EAAE,CAAC;QACtF,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QACtD,IAAI,MAAM,MAAM,CAAC,MAAM,CAAC;YAAE,MAAM,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;IAChH,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC;IACrG,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAClF,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QACrC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;QAC9F,CAAC,CAAC,MAAM,CAAC,KAAgC;QACzC,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE;QACzD,cAAc,EAAE,cAAc;QAC9B,IAAI,EAAE,0BAA0B;QAChC,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,QAAQ,EAAE,OAAO,CAAC,OAAO;QACzB,GAAG,CAAC,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,OAAO,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,MAA+B;IACrD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;QAC5F,CAAC,CAAC,MAAM,CAAC,KAAgC;QACzC,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,IAAI,GAAG,OAAO,KAAK,EAAE,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,+BAA+B,CAAC;IAC1G,MAAM,KAAK,GAAG,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC;IAC5F,MAAM,OAAO,GAAG,OAAO,KAAK,EAAE,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iDAAiD,CAAC;IACxI,OAAO,IAAI,UAAU,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,EAAE;QACvE,IAAI;QACJ,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAmB,CAAC,CAAC,CAAC,EAAE;KACnF,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,MAAc,EAAE,OAAoB;IAC9E,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,gCAAgC,CAAC,CAAC;IACrH,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,IAAI,IAAI,CAAC,cAAc,KAAK,cAAc,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO;WACpG,IAAI,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB;WACpG,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;QAChF,MAAM,WAAW,CAAC,sCAAsC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;WACpD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC;WACxE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,WAAW,CAAC,qCAAqC,CAAC,CAAC;IAC9G,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAiB,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAkB,EAAE,CAAC;AACnF,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,SAAiB,EACjB,OAAwB,EACxB,SAAiB,EACjB,KAAwB,EACxB,MAAgC;IAEhC,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE;QAC1D,cAAc,EAAE,cAAc;QAC9B,QAAQ,EAAE,OAAO,CAAC,OAAO;QACzB,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,UAAU,EAAE,OAAO,CAAC,SAAS;QAC7B,SAAS,EAAE,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC;QAChD,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI;QAC3B,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC;QACjB,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;QACzB,eAAe,EAAE,CAAC;QAClB,UAAU,EAAE,SAAS;KACtB,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,IAAY,EACZ,OAAwB,EACxB,SAAiB,EACjB,MAA0C,EAC1C,KAAwB,EACxB,MAAgC,EAChC,aAAgC,EAChC,cAAwC,EACxC,WAMI,EAAE;IAEN,MAAM,eAAe,CAAC,IAAI,EAAE;QAC1B,cAAc,EAAE,cAAc;QAC9B,QAAQ,EAAE,OAAO,CAAC,OAAO;QACzB,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,UAAU,EAAE,OAAO,CAAC,SAAS;QAC7B,SAAS,EAAE,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC;QAChD,MAAM;QACN,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC;QACjB,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;QACzB,cAAc,EAAE,CAAC,GAAG,aAAa,CAAC;QAClC,eAAe,EAAE,SAAS,CAAC,cAAc,CAAC;QAC1C,UAAU,EAAE,SAAS;QACrB,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,uBAAuB,EAAE,CAAC,GAAG,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,wBAAwB,EAAE,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,uBAAuB;IAC9B,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACjF,CAAC"}

package/dist/src/evals/progress.js

@@ -39,8 +39,8 @@ export function mergeEvalProgressTrial(progress, trial, now = new Date().toISOSt
             throw new TypeError(`eval progress trial identity conflict: ${parsed.trial_id}`);
         return progress;
     }
-    if (progress.trials.some((item) => item.run_id === parsed.run_id)) {
-        throw new TypeError(`eval progress run identity conflict: ${parsed.run_id}`);
+    if (progress.trials.some((item) => evalTrialCandidateKey(item) === evalTrialCandidateKey(parsed))) {
+        throw new TypeError(`eval progress run identity conflict: ${evalTrialCandidateKey(parsed)}`);
     }
     if (progress.trials.some((item) => evalTrialKey(item) === evalTrialKey(parsed))) {
         throw new TypeError(`eval progress logical trial conflict: ${parsed.task_id} attempt ${parsed.attempt}`);
@@ -64,6 +64,9 @@ export function mergeEvalProgressTrial(progress, trial, now = new Date().toISOSt
 export function evalTrialKey(trial) {
     return `${trial.task_id}\u0000${trial.attempt}`;
 }
+export function evalTrialCandidateKey(trial) {
+    return trial.run_group ? trial.run_group.run_group_id : trial.run_id;
+}
 /** Replace one invalid logical trial, or fill a missing slot, with a valid verifier result. */
 export function replaceInvalidEvalProgressTrial(progress, trial, now = new Date().toISOString()) {
     const parsed = parseEvalTrialRef(trial, "eval rerun trial");
@@ -76,8 +79,8 @@ export function replaceInvalidEvalProgressTrial(progress, trial, now = new Date(
             return progress;
         throw new TypeError(`eval rerun cannot replace valid task: ${parsed.task_id}`);
     }
-    if (progress.trials.some((item) => item.run_id === parsed.run_id && evalTrialKey(item) !== key)) {
-        throw new TypeError(`eval progress run identity conflict: ${parsed.run_id}`);
+    if (progress.trials.some((item) => evalTrialCandidateKey(item) === evalTrialCandidateKey(parsed) && evalTrialKey(item) !== key)) {
+        throw new TypeError(`eval progress run identity conflict: ${evalTrialCandidateKey(parsed)}`);
     }
     if (progress.trials.some((item) => item.trial_id === parsed.trial_id && evalTrialKey(item) !== key)) {
         throw new TypeError(`eval progress trial identity conflict: ${parsed.trial_id}`);
@@ -120,7 +123,7 @@ export function parseEvalProgress(value) {
         throw new TypeError("eval progress trials are invalid");
     const trials = record.trials.map((trial, index) => parseEvalTrialRef(trial, `eval progress trial ${index}`));
     if (new Set(trials.map((trial) => trial.trial_id)).size !== trials.length
-        || new Set(trials.map((trial) => trial.run_id)).size !== trials.length)
+        || new Set(trials.map(evalTrialCandidateKey)).size !== trials.length)
         throw new TypeError("eval progress trial identities are duplicated");
     const sorted = [...trials].sort((left, right) => left.task_id.localeCompare(right.task_id)
         || left.attempt - right.attempt
@@ -169,7 +172,6 @@ export function parseEvalTrialRef(value, label = "eval trial") {
         throw new TypeError(`${label} must be an object`);
     const trial = value;
     if (typeof trial.trial_id !== "string" || !trial.trial_id
-        || typeof trial.run_id !== "string" || !RUN_ID.test(trial.run_id)
         || typeof trial.task_id !== "string" || !trial.task_id
         || !Number.isSafeInteger(trial.attempt) || trial.attempt < 1
         || (trial.observation_status !== "valid" && trial.observation_status !== "invalid")) {
@@ -178,21 +180,59 @@ export function parseEvalTrialRef(value, label = "eval trial") {
     if (trial.observation_status === "valid" && (typeof trial.reward !== "number" || !Number.isFinite(trial.reward))) {
         throw new TypeError(`${label} valid reward is invalid`);
     }
+    const scores = trial.scores === undefined ? undefined : parseScores(trial.scores, label);
+    if (scores !== undefined && (trial.observation_status !== "valid" || scores.total_score !== trial.reward)) {
+        throw new TypeError(`${label} scores do not match the valid reward`);
+    }
     if (trial.observation_status === "invalid" && (typeof trial.invalid_reason !== "string" || !trial.invalid_reason)) {
         throw new TypeError(`${label} invalid reason is missing`);
     }
     if (trial.verifier_result_ref !== undefined && (typeof trial.verifier_result_ref !== "string" || !trial.verifier_result_ref)) {
         throw new TypeError(`${label} verifier ref is invalid`);
     }
-    return {
+    const assessment = trial.assessment;
+    if (assessment !== undefined && (!assessment || typeof assessment.id !== "string" || !/^assessment_[a-f0-9]{32}$/.test(assessment.id)
+        || typeof assessment.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(assessment.digest)))
+        throw new TypeError(`${label} assessment is invalid`);
+    const group = trial.run_group;
+    if (group !== undefined) {
+        if (!group || typeof group !== "object" || Array.isArray(group) || Object.keys(group).some(key => !["run_group_id", "digest"].includes(key))
+            || typeof group.run_group_id !== "string" || !/^run_group_[a-f0-9]{32}$/.test(group.run_group_id)
+            || typeof group.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(group.digest) || trial.run_id !== undefined || !assessment) {
+            throw new TypeError(`${label} phase group identity is invalid`);
+        }
+    }
+    else if (typeof trial.run_id !== "string" || !RUN_ID.test(trial.run_id))
+        throw new TypeError(`${label} run identity is invalid`);
+    const common = {
         trial_id: trial.trial_id,
-        run_id: trial.run_id,
         task_id: trial.task_id,
         attempt: trial.attempt,
         observation_status: trial.observation_status,
         ...(trial.reward === undefined ? {} : { reward: trial.reward }),
+        ...(scores === undefined ? {} : { scores }),
         ...(trial.verifier_result_ref === undefined ? {} : { verifier_result_ref: trial.verifier_result_ref }),
         ...(trial.invalid_reason === undefined ? {} : { invalid_reason: trial.invalid_reason }),
     };
+    const reference = assessment ? { id: assessment.id, digest: assessment.digest } : undefined;
+    return group ? { ...common, run_group: { run_group_id: group.run_group_id, digest: group.digest }, assessment: reference }
+        : { ...common, run_id: trial.run_id, ...(reference ? { assessment: reference } : {}) };
+}
+function parseScores(value, label) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError(`${label} scores are invalid`);
+    const record = value;
+    if (Object.keys(record).some((key) => !["total_score", "process_score", "normalization"].includes(key))
+        || typeof record.total_score !== "number" || !Number.isFinite(record.total_score)
+        || record.process_score !== undefined && (typeof record.process_score !== "number" || !Number.isFinite(record.process_score))
+        || record.normalization !== "standard" && record.normalization !== "legacy-reward"
+        || record.normalization === "legacy-reward" && record.process_score !== undefined) {
+        throw new TypeError(`${label} scores are invalid`);
+    }
+    return {
+        total_score: record.total_score,
+        ...(record.process_score === undefined ? {} : { process_score: record.process_score }),
+        normalization: record.normalization,
+    };
 }
 //# sourceMappingURL=progress.js.map