agent-hitch 0.2.6 → 0.2.8

package/dist/src/evals/planned-progress-publisher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-progress-publisher.js","sourceRoot":"","sources":["../../../src/evals/planned-progress-publisher.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC3G,OAAO,EAAE,2BAA2B,EAAE,MAAM,iCAAiC,CAAC;AAG9E,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAiB;IACzB,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,OAAO,CAA8B;IAEtD,YAAY,QAAwB,EAAE,OAAoC;QACxE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,OAAO,CAAC,GAAmB,EAAE,MAAc;QACzC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED,cAAc,CAAC,GAAmB,EAAE,MAAc;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACtD,CAAC;IAEO,OAAO,CAAC,GAAmB,EAAE,MAAc,EAAE,IAAkC;QACrF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAC1C,MAAM,IAAI,GAAG,IAAI,KAAK,QAAQ;gBAC5B,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;gBAC5C,CAAC,CAAC,+BAA+B,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YACxD,IAAI,IAAI,CAAC,UAAU,KAAK,QAAQ;gBAAE,OAAO;YACzC,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBAC/E,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY;gBAC9C,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB;aAC3D,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,SAAS;gBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;iBAC7L,CAAC;gBACJ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAA6B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBACjI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;YAC7J,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,MAAM,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACrB,IAAI,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB;gBACxE,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM;gBAC3C,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;aACrC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,IAAI,CAAC;IAClB,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF"}

package/dist/src/evals/planned-remote-execution.js

@@ -0,0 +1,40 @@
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+import { loadTrialEnvironmentImages } from "./trial-environment-evidence.js";
+export async function executeRemotePlannedWorkItem(input) {
+    const { options, item } = input;
+    if (!options.remoteWorkExecutor)
+        throw new TypeError("remote work executor is unavailable");
+    const taskId = item.task_ids[0];
+    const environmentImages = await loadTrialEnvironmentImages({
+        taskId, uses: item.image_refs ?? [],
+        ...(options.environmentImageManifestLoader ? { loader: options.environmentImageManifestLoader } : {}),
+    });
+    const completed = await options.remoteWorkExecutor({
+        evalId: options.evalId,
+        evalDirectory: options.evalDirectory,
+        root: options.root,
+        request: options.request,
+        plan: options.plan,
+        workItem: item,
+        resolvedRevision: options.resolvedRevision,
+        preparedArtifact: preparedArtifactForWorkItem(options, item),
+        runtimeDirectory: options.controllerRuntime.directory,
+        runtimeId: options.controllerRuntime.runtime_id,
+        ...(environmentImages ? { environmentImages } : {}),
+        ...(options.plan.model_capture ? { modelCapturePlan: options.plan.model_capture } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+        emit: (event) => options.sink.emit(event),
+        publish: input.publish,
+        onLeaseState: (leaseId, state) => options.onWorkItemState?.(item.work_id, leaseId, state) ?? Promise.resolve(),
+    });
+    return {
+        attempt: item.logical_attempt,
+        workId: item.work_id,
+        tasks: [taskId],
+        refs: completed.refs,
+        leaseId: completed.leaseId,
+        run: completed.run,
+        ...(environmentImages ? { environmentImages } : {}),
+    };
+}
+//# sourceMappingURL=planned-remote-execution.js.map

package/dist/src/evals/planned-remote-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-remote-execution.js","sourceRoot":"","sources":["../../../src/evals/planned-remote-execution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAIlD;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,IAAI,CAAC,OAAO,CAAC,kBAAkB;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC;QACzD,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;QACnC,GAAG,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,8BAA8B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtG,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,kBAAkB,CAAC;QACjD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,gBAAgB,EAAE,2BAA2B,CAAC,OAAO,EAAE,IAAI,CAAC;QAC5D,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;QACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;QAC/C,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACzC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE;KAC/G,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,eAAyB;QACvC,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,KAAK,EAAE,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,GAAG,EAAE,SAAS,CAAC,GAAG;QAClB,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAC;AACJ,CAAC"}

package/dist/src/evals/planned-retry-execution.js

@@ -0,0 +1,174 @@
+import path from "node:path";
+import { runInfrastructureRetries } from "./infrastructure-retry.js";
+import { runRemoteInfrastructureRetries } from "./remote-infrastructure-retry.js";
+import { beginPlannedInfrastructureRetry } from "./planned-retry-lifecycle.js";
+import { localSourceBackendFailure } from "./result-helpers.js";
+import { resourceRequirementForTask } from "./execution-plan-resources.js";
+import { resolvedImageMapping } from "./environment-image-planning.js";
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+import { workSchedulingPriority } from "./planned-execution-support.js";
+import { ensurePhysicalRetryDecision, ensureTerminalRetryDecision, readEvalRetryState } from "./retry-state.js";
+import { classifyTrialFailure, physicalRetryAllowed } from "./failure-classifier.js";
+import { physicalRetryWorkItem } from "./physical-retry-work.js";
+import { retryBackoffMs } from "./retry-backoff.js";
+export async function runPlannedInfrastructureRetries(options, publisher, semaphore, metrics, workItem, completed) {
+    if (options.signal?.aborted || completed.run.backend.process_exit_code !== 0 || completed.run.rawResult === null)
+        return [];
+    if (options.localTransport && localSourceBackendFailure(completed.run.rawResult))
+        return [];
+    for (const trigger of completed.refs.filter((ref) => ref.observation_status === "invalid")) {
+        const classification = classifyTrialFailure(trigger);
+        if (physicalRetryAllowed(classification) && options.request.infrastructure_retries > 0)
+            continue;
+        const decision = await ensureTerminalRetryDecision({
+            evalDirectory: options.evalDirectory, evalId: options.evalId, item: workItem, retryIndex: 1, trigger,
+            exhausted: physicalRetryAllowed(classification),
+        });
+        if (decision)
+            options.sink.emit({
+                type: "eval.retry.skipped", work_id: workItem.work_id, decision_id: decision.decision_id,
+                task_id: trigger.task_id, attempt: trigger.attempt, disposition: decision.disposition,
+                classification: decision.classification, state: decision.state,
+            });
+        if (decision && providerCircuitFailure(decision.classification.code))
+            options.sink.emit({
+                type: "eval.provider-circuit.opened",
+                scope: "trial-retry",
+                mode: "retry-only",
+                provider: workItem.provider,
+                model: options.request.model || null,
+                work_id: workItem.work_id,
+                decision_id: decision.decision_id,
+                stable_code: decision.classification.code,
+                automatic_probe: false,
+            });
+    }
+    return runPlannedInfrastructureRetriesFromSource(options, publisher, semaphore, metrics, workItem, completed, 1);
+}
+function providerCircuitFailure(code) {
+    return code === "provider_quota_exhausted"
+        || code === "provider_auth_failed"
+        || code === "provider_configuration_invalid";
+}
+export async function runPlannedInfrastructureRetriesFromSource(options, publisher, semaphore, metrics, workItem, source, firstRetryIndex) {
+    const progress = publisher.current();
+    if (options.remoteWorkExecutor) {
+        return (await runRemoteInfrastructureRetries({
+            options, item: workItem, initial: source, progress, firstRetryIndex, metrics,
+            replaceProgressTrial: (ref, retryWorkId) => publisher.replaceInvalid(ref, retryWorkId),
+            currentProgress: () => publisher.current(),
+        })).runs;
+    }
+    const retries = await runInfrastructureRetries({
+        evalId: options.evalId,
+        evalDirectory: options.evalDirectory,
+        backendBaseDirectory: path.join(options.evalDirectory, "harbor", "work-items", source.workId, "infrastructure-retries"),
+        logicalAttempt: source.attempt,
+        initialRefs: source.refs,
+        progress,
+        request: options.request,
+        root: options.root,
+        resolvedRevision: options.resolvedRevision,
+        controllerRuntime: options.controllerRuntime,
+        preparedArtifact: preparedArtifactForWorkItem(options, workItem),
+        executionResources: resourceRequirementForTask(options.plan, source.tasks[0])?.main_limits ?? options.plan.default_trial_resources,
+        resolvedImages: resolvedImageMapping(workItem.image_refs ?? []),
+        ...(source.environmentImages ? { environmentImages: source.environmentImages } : {}),
+        beginRetry: async ({ retry, triggers, backendDirectory }) => {
+            const directRelease = await semaphore?.acquire(options.signal, workSchedulingPriority(workItem));
+            try {
+                const lifecycle = await beginPlannedInfrastructureRetry({
+                    options, item: workItem, retry, triggers, backendDirectory,
+                    ...(source.environmentImages ? { environmentImages: source.environmentImages } : {}),
+                });
+                return {
+                    ...lifecycle,
+                    close: async () => { try {
+                        await lifecycle.close();
+                    }
+                    finally {
+                        directRelease?.();
+                    } },
+                };
+            }
+            catch (error) {
+                directRelease?.();
+                throw error;
+            }
+        },
+        replaceProgressTrial: (ref, retryWorkId) => publisher.replaceInvalid(ref, retryWorkId),
+        currentProgress: () => publisher.current(),
+        originWorkItem: workItem,
+        firstRetryIndex,
+        onRetryExecutionStarted: (workId) => metrics.startWork(workId, "retry"),
+        onRetryExecutionFinished: (workId) => metrics.finishWork(workId),
+        onRetryBackoff: (durationMs) => metrics.addBlocked("backoff", durationMs),
+        onVerifierDuration: (durationMs) => metrics.addVerifier(durationMs),
+        ...(options.onWorkItemQueued ? { onRetryWorkPlanned: options.onWorkItemQueued } : {}),
+        ...(options.localTransport ? { localTransport: options.localTransport } : {}),
+        ...(options.plan.model_capture ? { modelCapturePlan: options.plan.model_capture } : {}),
+        ...(options.interactionCaptureExporter ? { interactionCaptureExporter: options.interactionCaptureExporter } : {}),
+        env: options.env,
+        ...(options.harborExecutable !== undefined ? { harborExecutable: options.harborExecutable } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(options.trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs: options.trialBundleGraceMs }),
+        sink: options.sink,
+        ...(options.localTransport ? { stopAfterResult: localSourceBackendFailure } : {}),
+    });
+    return retries.runs;
+}
+export async function reconcilePersistedInitialRetryDecisions(options) {
+    let state = await readEvalRetryState(options.evalDirectory, options.evalId);
+    for (const trigger of options.progress.trials.filter((trial) => trial.observation_status === "invalid")) {
+        if (state?.decisions.some((decision) => decision.trigger_trial_id === trigger.trial_id))
+            continue;
+        const item = options.plan.work_items.find((entry) => entry.logical_attempt === trigger.attempt && entry.task_ids.includes(trigger.task_id));
+        if (!item)
+            throw new TypeError(`persisted invalid trial is absent from execution plan: ${trigger.task_id}#${trigger.attempt}`);
+        const classification = classifyTrialFailure(trigger);
+        if (physicalRetryAllowed(classification) && options.request.infrastructure_retries > 0) {
+            const retryWork = physicalRetryWorkItem(item, 1, [trigger]);
+            await ensurePhysicalRetryDecision({
+                evalDirectory: options.evalDirectory, evalId: options.evalId, item, retryIndex: 1, trigger,
+                notBefore: new Date(Date.now() + retryBackoffMs(options.request.infrastructure_retry_backoff_ms, 1, retryWork.work_id)).toISOString(),
+            });
+        }
+        else {
+            await ensureTerminalRetryDecision({
+                evalDirectory: options.evalDirectory, evalId: options.evalId, item, retryIndex: 1, trigger,
+                exhausted: physicalRetryAllowed(classification),
+            });
+        }
+        state = await readEvalRetryState(options.evalDirectory, options.evalId);
+    }
+}
+export async function pendingRetryWork(options) {
+    const state = await readEvalRetryState(options.evalDirectory, options.evalId);
+    const pending = new Map();
+    if (!state)
+        return pending;
+    const seen = new Set();
+    for (const decision of state.decisions.filter((entry) => entry.disposition === "physical-retry" && entry.state === "planned")) {
+        if (!decision.retry_work_id || seen.has(decision.retry_work_id))
+            continue;
+        seen.add(decision.retry_work_id);
+        const item = options.plan.work_items.find((entry) => entry.slots.includes(decision.slot_id));
+        if (!item || item.logical_attempt === null || item.task_ids.length !== 1)
+            throw new TypeError(`retry decision slot is absent from execution plan: ${decision.slot_id}`);
+        const persistedTrigger = options.progress.trials.find((trial) => trial.trial_id === decision.trigger_trial_id);
+        const trigger = persistedTrigger ?? (decision.trigger_run_id ? {
+            trial_id: decision.trigger_trial_id, run_id: decision.trigger_run_id, task_id: item.task_ids[0],
+            attempt: item.logical_attempt, observation_status: "invalid", invalid_reason: decision.classification.code,
+        } : undefined);
+        if (!trigger)
+            throw new TypeError(`retry decision trigger cannot be recovered: ${decision.decision_id}`);
+        const taskId = item.task_ids[0];
+        const entries = pending.get(taskId) ?? [];
+        entries.push({ item, decision, trigger });
+        pending.set(taskId, entries);
+    }
+    for (const entries of pending.values())
+        entries.sort((left, right) => left.decision.retry_index - right.decision.retry_index);
+    return pending;
+}
+//# sourceMappingURL=planned-retry-execution.js.map

package/dist/src/evals/planned-retry-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-retry-execution.js","sourceRoot":"","sources":["../../../src/evals/planned-retry-execution.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAErE,OAAO,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,+BAA+B,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AAEvE,OAAO,EAAE,sBAAsB,EAA0B,MAAM,gCAAgC,CAAC;AAKhG,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEhH,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAQpD,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,OAAoC,EACpC,SAA4B,EAC5B,SAAwC,EACxC,OAA6B,EAC7B,QAA2B,EAC3B,SAA4B;IAE5B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5H,IAAI,OAAO,CAAC,cAAc,IAAI,yBAAyB,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5F,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QAC3F,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,oBAAoB,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC;YAAE,SAAS;QACjG,MAAM,QAAQ,GAAG,MAAM,2BAA2B,CAAC;YACjD,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO;YACpG,SAAS,EAAE,oBAAoB,CAAC,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,IAAI,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC9B,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACxF,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACrF,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK;aAC/D,CAAC,CAAC;QACH,IAAI,QAAQ,IAAI,sBAAsB,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtF,IAAI,EAAE,8BAA8B;gBACpC,KAAK,EAAE,aAAa;gBACpB,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI;gBACpC,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,WAAW,EAAE,QAAQ,CAAC,cAAc,CAAC,IAAI;gBACzC,eAAe,EAAE,KAAK;aACvB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,yCAAyC,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;AACnH,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI,KAAK,0BAA0B;WACrC,IAAI,KAAK,sBAAsB;WAC/B,IAAI,KAAK,gCAAgC,CAAC;AACjD,CAAC;AAUD,MAAM,CAAC,KAAK,UAAU,yCAAyC,CAC7D,OAAoC,EACpC,SAA4B,EAC5B,SAAwC,EACxC,OAA6B,EAC7B,QAA2B,EAC3B,MAAmB,EACnB,eAAuB;IAEvB,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;IACrC,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,OAAO,CAAC,MAAM,8BAA8B,CAAC;YAC3C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO;YAC5E,oBAAoB,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC;YACtF,eAAe,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE;SAC3C,CAAC,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,wBAAwB,CAAC;QAC7C,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,wBAAwB,CAAC;QACvH,cAAc,EAAE,MAAM,CAAC,OAAO;QAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;QACxB,QAAQ;QACR,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,gBAAgB,EAAE,2BAA2B,CAAC,OAAO,EAAE,QAAQ,CAAC;QAChE,kBAAkB,EAAE,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAW,CAAC,EAAE,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,uBAAuB;QAC5I,cAAc,EAAE,oBAAoB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;QAC/D,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,UAAU,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAC1D,MAAM,aAAa,GAAG,MAAM,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjG,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,+BAA+B,CAAC;oBACtD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB;oBAC1D,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACrF,CAAC,CAAC;gBACH,OAAO;oBACL,GAAG,SAAS;oBACZ,KAAK,EAAE,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC;wBAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;oBAAC,CAAC;4BAAS,CAAC;wBAAC,aAAa,EAAE,EAAE,CAAC;oBAAC,CAAC,CAAC,CAAC;iBACvF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE,CAAC;gBAClB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC;QACtF,eAAe,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE;QAC1C,cAAc,EAAE,QAAQ;QACxB,eAAe;QACf,uBAAuB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC;QACvE,wBAAwB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;QAChE,cAAc,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC;QACzE,kBAAkB,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC;QACnE,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7E,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,0BAA0B,EAAE,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACvG,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,yBAAyB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClF,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,IAAI,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uCAAuC,CAAC,OAAoC;IAChG,IAAI,KAAK,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QACxG,IAAI,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,KAAK,OAAO,CAAC,QAAQ,CAAC;YAAE,SAAS;QAClG,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,KAAK,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5I,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,0DAA0D,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/H,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,oBAAoB,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC,EAAE,CAAC;YACvF,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5D,MAAM,2BAA2B,CAAC;gBAChC,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO;gBAC1F,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE;aACtI,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,2BAA2B,CAAC;gBAChC,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO;gBAC1F,SAAS,EAAE,oBAAoB,CAAC,cAAc,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;QACD,KAAK,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAoC;IACzE,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,IAAI,CAAC,KAAK;QAAE,OAAO,OAAO,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,gBAAgB,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;QAC9H,IAAI,CAAC,QAAQ,CAAC,aAAa,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,SAAS;QAC1E,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,sDAAsD,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACxK,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QAC/G,MAAM,OAAO,GAAG,gBAAgB,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;YAC7D,QAAQ,EAAE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,EAAE,QAAQ,CAAC,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW;YACzG,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,kBAAkB,EAAE,SAAkB,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,IAAI;SACpH,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACf,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzG,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;QAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9H,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/evals/planned-retry-lifecycle.js

@@ -0,0 +1,188 @@
+import { dockerResourceOwnership } from "./docker-ownership.js";
+import { startDockerResourceObserver } from "./docker-resource-observer.js";
+import { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, DEFAULT_EXECUTION_LEASE_TTL_MS, createExecutionLease } from "./execution-leases.js";
+import { resourceRequirementForTask, runtimeResourcesForTask } from "./execution-plan-resources.js";
+import { recordLocalDockerProcessExit, recordLocalDockerProcessStart, releaseLocalDockerProcessRecord } from "./local-docker-provider.js";
+import { resolvedImageMapping } from "./environment-image-planning.js";
+import { prebuiltTaskImage, verifyTrialEnvironmentImageExecution } from "./trial-environment-evidence.js";
+import { physicalRetryWorkItem } from "./physical-retry-work.js";
+export async function beginPlannedInfrastructureRetry(input) {
+    const { options } = input;
+    const item = physicalRetryWorkItem(input.item, input.retry, input.triggers);
+    const taskId = item.task_ids[0];
+    let permit;
+    let lease;
+    let runningAnnounced = false;
+    try {
+        permit = await options.admission?.acquire({
+            evalId: options.evalId,
+            workItem: item,
+            maxParallelism: options.plan.max_parallelism,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        if (permit)
+            options.sink.emit({
+                type: "eval.physical-retry.admitted",
+                execution_kind: "physical-infrastructure-retry",
+                work_id: item.work_id,
+                retry: input.retry,
+                allocation_id: permit.allocationId,
+                collision_keys: permit.collisionKeys,
+                reservation: item.reservation,
+            });
+        lease = await createExecutionLease({
+            evalDirectory: options.evalDirectory,
+            evalId: options.evalId,
+            workId: item.work_id,
+            worker: permit ? { ...options.worker, parentAllocationId: permit.allocationId } : options.worker,
+            reservation: item.reservation,
+            ttlMs: DEFAULT_EXECUTION_LEASE_TTL_MS,
+        });
+        const epoch = lease.current().epoch;
+        await lease.markRunning(epoch);
+        await options.onWorkItemState?.(item.work_id, lease.leaseId, "running");
+        runningAnnounced = true;
+        options.sink.emit({
+            type: "eval.physical-retry.started",
+            execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true,
+            work_id: item.work_id,
+            lease_id: lease.leaseId,
+            lease_epoch: epoch,
+            retry: input.retry,
+            task_id: taskId,
+            attempt: item.logical_attempt,
+            trigger_trials: input.triggers.map((trial) => trial.trial_id).sort(),
+        });
+        return retryLifecycle({ ...input, item }, lease, permit, runningAnnounced);
+    }
+    catch (error) {
+        if (lease)
+            await lease.release(lease.current().epoch).catch(() => undefined);
+        if (runningAnnounced)
+            await options.onWorkItemState?.(item.work_id, lease?.leaseId ?? "", "terminal").catch(() => undefined);
+        permit?.release();
+        throw error;
+    }
+}
+function retryLifecycle(input, lease, permit, runningAnnounced) {
+    const { options, item } = input;
+    const taskId = item.task_ids[0];
+    const epoch = lease.current().epoch;
+    const runtime = runtimeResourcesForTask(options.plan, taskId, item.reservation);
+    const ownership = dockerResourceOwnership(options.root, lease.current(), taskId);
+    const observer = startDockerResourceObserver({
+        ownership,
+        workerId: options.worker.workerId,
+        collisionDomainId: options.worker.collisionDomainId,
+        reservation: item.reservation,
+        mainLimits: runtime.mainLimits,
+        sidecarLimits: runtime.sidecarLimits,
+        env: options.env,
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    let processRecorded = false;
+    let heartbeatFailure;
+    let heartbeatTail = Promise.resolve();
+    const timer = setInterval(() => {
+        if (heartbeatFailure !== undefined)
+            return;
+        heartbeatTail = heartbeatTail.then(async () => {
+            const renewed = await lease.heartbeat(epoch);
+            options.sink.emit({
+                type: "lease.renewed",
+                execution_kind: "physical-infrastructure-retry",
+                work_id: item.work_id,
+                lease_id: lease.leaseId,
+                lease_epoch: renewed.epoch,
+                heartbeat_at: renewed.heartbeat_at,
+                expires_at: renewed.expires_at,
+            });
+        }).catch((error) => { heartbeatFailure ??= error; });
+    }, DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS);
+    timer.unref();
+    let closed;
+    const prebuiltImage = prebuiltTaskImage(input.environmentImages);
+    const close = () => closed ??= closeRetryLifecycle({ input, lease, permit, observer, timer, heartbeatTail: () => heartbeatTail, heartbeatFailure: () => heartbeatFailure, processRecorded: () => processRecorded, runningAnnounced });
+    return {
+        leaseId: lease.leaseId,
+        workId: item.work_id,
+        ...(input.environmentImages ? { environmentImages: input.environmentImages } : {}),
+        captureExecutionEvidence: async () => {
+            if (heartbeatFailure !== undefined)
+                throw heartbeatFailure;
+            return verifyTrialEnvironmentImageExecution(await observer.capture(), input.environmentImages);
+        },
+        backend: {
+            executionResources: resourceRequirementForTask(options.plan, taskId)?.main_limits ?? options.plan.default_trial_resources,
+            ...(Object.keys(runtime.sidecarLimits).length > 0 ? { dockerServiceLimits: runtime.sidecarLimits } : {}),
+            dockerOwnership: ownership,
+            resolvedImages: resolvedImageMapping(item.image_refs ?? []),
+            ...(prebuiltImage ? { prebuiltTaskImage: prebuiltImage } : {}),
+            ...(options.worker.provider === "local-docker" && process.platform !== "win32" ? {
+                recoverableProcess: true,
+                onProcessStarted: (pid) => recordLocalDockerProcessStart({
+                    root: options.root,
+                    workerId: options.worker.workerId,
+                    evalDirectory: options.evalDirectory,
+                    lease: lease.current(),
+                    backendDirectory: input.backendDirectory,
+                    pid,
+                }).then(() => { processRecorded = true; }),
+                onProcessExited: (result) => recordLocalDockerProcessExit({
+                    root: options.root,
+                    leaseId: lease.leaseId,
+                    epoch,
+                    ...result,
+                }).then(() => undefined),
+            } : {}),
+        },
+        close,
+    };
+}
+async function closeRetryLifecycle(input) {
+    const { options, item } = input.input;
+    const epoch = input.lease.current().epoch;
+    let failure;
+    const attempt = async (operation) => {
+        try {
+            await operation();
+        }
+        catch (error) {
+            failure ??= error;
+        }
+    };
+    clearInterval(input.timer);
+    await attempt(() => input.observer.stop());
+    await attempt(input.heartbeatTail);
+    failure ??= input.heartbeatFailure();
+    if (input.processRecorded())
+        await attempt(() => releaseLocalDockerProcessRecord({ root: options.root, leaseId: input.lease.leaseId, epoch }));
+    await attempt(async () => {
+        const released = await input.lease.release(epoch);
+        options.sink.emit({
+            type: "eval.physical-retry.lease-released",
+            execution_kind: "physical-infrastructure-retry",
+            work_id: item.work_id,
+            lease_id: input.lease.leaseId,
+            lease_epoch: released.epoch,
+            retry: input.input.retry,
+            state: released.state,
+        });
+    });
+    if (options.dockerResourceReaper) {
+        try {
+            const report = await options.dockerResourceReaper({ root: options.root, leaseIds: [input.lease.leaseId], env: options.env });
+            options.sink.emit({ type: "eval.physical-retry.resources-reaped", work_id: item.work_id, lease_id: input.lease.leaseId, retry: input.input.retry, scanned: report.scanned, deleted: report.deleted.length, issues: report.issues.length });
+        }
+        catch (error) {
+            options.sink.emit({ type: "eval.physical-retry.reaper-failed", work_id: item.work_id, lease_id: input.lease.leaseId, retry: input.input.retry, code: error.code || "docker_reaper_failed" });
+        }
+    }
+    if (input.runningAnnounced)
+        await attempt(() => options.onWorkItemState?.(item.work_id, input.lease.leaseId, "terminal") ?? Promise.resolve());
+    input.permit?.release();
+    if (failure !== undefined)
+        throw failure;
+}
+//# sourceMappingURL=planned-retry-lifecycle.js.map

package/dist/src/evals/planned-retry-lifecycle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-retry-lifecycle.js","sourceRoot":"","sources":["../../../src/evals/planned-retry-lifecycle.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAEnI,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAEpG,OAAO,EAAE,4BAA4B,EAAE,6BAA6B,EAAE,+BAA+B,EAAE,MAAM,4BAA4B,CAAC;AAE1I,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,oCAAoC,EAAE,MAAM,iCAAiC,CAAC;AAE1G,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,KAOrD;IACC,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC1B,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,IAAI,MAAyF,CAAC;IAC9F,IAAI,KAAuC,CAAC;IAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe;YAC5C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,8BAA8B;gBACpC,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,aAAa,EAAE,MAAM,CAAC,YAAY;gBAClC,cAAc,EAAE,MAAM,CAAC,aAAa;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACH,KAAK,GAAG,MAAM,oBAAoB,CAAC;YACjC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YAChG,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,8BAA8B;SACtC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;QACpC,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACxE,gBAAgB,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,6BAA6B;YACnC,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,IAAI,CAAC,eAAe;YAC7B,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;SACrE,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7E,IAAI,gBAAgB;YAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7H,MAAM,EAAE,OAAO,EAAE,CAAC;QAClB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,KAMC,EACD,KAA2B,EAC3B,MAAyG,EACzG,gBAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IACpC,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,2BAA2B,CAAC;QAC3C,SAAS;QACT,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;QACjC,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB;QACnD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,gBAAyB,CAAC;IAC9B,IAAI,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,gBAAgB,KAAK,SAAS;YAAE,OAAO;QAC3C,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,eAAe;gBACrB,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;gBACvB,WAAW,EAAE,OAAO,CAAC,KAAK;gBAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,UAAU,EAAE,OAAO,CAAC,UAAU;aAC/B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,MAAiC,CAAC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,GAAkB,EAAE,CAAC,MAAM,KAAK,mBAAmB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACrP,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,wBAAwB,EAAE,KAAK,IAAI,EAAE;YACnC,IAAI,gBAAgB,KAAK,SAAS;gBAAE,MAAM,gBAAgB,CAAC;YAC3D,OAAO,oCAAoC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,EAAE;YACP,kBAAkB,EAAE,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,uBAAuB;YACzH,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,eAAe,EAAE,SAAS;YAC1B,cAAc,EAAE,oBAAoB,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAC3D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC;gBAC/E,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,6BAA6B,CAAC;oBAC/D,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;oBACjC,aAAa,EAAE,OAAO,CAAC,aAAa;oBACpC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;oBACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;oBACxC,GAAG;iBACJ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC1C,eAAe,EAAE,CAAC,MAA8D,EAAE,EAAE,CAAC,4BAA4B,CAAC;oBAChH,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;oBACL,GAAG,MAAM;iBACV,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;aACzB,CAAC,CAAC,CAAC,EAAE,CAAC;SACR;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAUlC;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IAC1C,IAAI,OAAgB,CAAC;IACrB,MAAM,OAAO,GAAG,KAAK,EAAE,SAAiC,EAAiB,EAAE;QACzE,IAAI,CAAC;YAAC,MAAM,SAAS,EAAE,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAK,CAAC;QAAC,CAAC;IACjE,CAAC,CAAC;IACF,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,gBAAgB,EAAE,CAAC;IACrC,IAAI,KAAK,CAAC,eAAe,EAAE;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,+BAA+B,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/I,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,oCAAoC;YAC1C,cAAc,EAAE,+BAA+B;YAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO;YAC7B,WAAW,EAAE,QAAQ,CAAC,KAAK;YAC3B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;YACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7H,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,sCAAsC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7O,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mCAAmC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAG,KAA2B,CAAC,IAAI,IAAI,sBAAsB,EAAE,CAAC,CAAC;QACtN,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,gBAAgB;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/I,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IACxB,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC"}