intentdna 1.9.0 → 1.9.3

package/dist/runtime/push-driver.js

@@ -0,0 +1,1002 @@
+import { createHash, randomUUID } from "node:crypto";
+import { canonicalizeJson } from "./canonical-json.js";
+import { parseStoredExecutableRunPlan } from "./executable-run-plan.js";
+import { CALLER_ATTESTATION_SCHEMA_VERSION } from "./run-contracts.js";
+import { parseStoredRunBinding, } from "./run-binding.js";
+import { executeWorkerAttempt, } from "./worker-executor.js";
+export class PushDriverError extends Error {
+    code;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = "PushDriverError";
+        this.code = code;
+    }
+}
+export class PushDriverIndeterminateAttachError extends PushDriverError {
+    disposition = "retain_preparation";
+    constructor(attemptId, cause) {
+        super("attach_indeterminate", `attempt ${attemptId} attach disposition is indeterminate; execution authority preparation was retained`, { cause });
+        this.name = "PushDriverIndeterminateAttachError";
+    }
+}
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000;
+const DEFAULT_WAIT_POLL_MS = 50;
+const NO_ATTEMPT_ERROR = Symbol("no_attempt_error");
+const DEFAULT_CAPABILITIES = {
+    worktree_isolation: true,
+    parallelism: true,
+    cancellation: false,
+    hook_identity: false,
+    caller_attestation: false,
+    human_interaction: false,
+    direct_structured_submission: true,
+    network_policy_enforcement: false,
+};
+const RECOVERABLE_WAIT_REASONS = new Set([
+    "attempt_active",
+    "dependencies_pending",
+    "retry_backoff",
+    "run_cancelling",
+]);
+function defaultSleep(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+function requirePositiveInteger(name, value) {
+    if (!Number.isSafeInteger(value) || value <= 0) {
+        throw new PushDriverError("invalid_configuration", `${name} must be a positive safe integer`);
+    }
+}
+function requireNonEmpty(name, value) {
+    if (value.length === 0) {
+        throw new PushDriverError("invalid_service_response", `${name} must not be empty`);
+    }
+}
+function assertRequiredCapabilities(required, provided) {
+    for (const [name, needed] of Object.entries(required)) {
+        if (needed && provided[name] !== true) {
+            throw new PushDriverError("unsupported", `driver capability ${name} is required`);
+        }
+    }
+}
+function defaultWaitStrategy(waitPollMs) {
+    return ({ reason_code }) => RECOVERABLE_WAIT_REASONS.has(reason_code)
+        ? { action: "continue", delay_ms: waitPollMs }
+        : { action: "stop", reason: `unsupported wait reason: ${reason_code}` };
+}
+function validateRunPayloads(view) {
+    try {
+        const plan = parseStoredExecutableRunPlan(view.plan);
+        const binding = parseStoredRunBinding(view.run.run_binding);
+        if (view.run.plan_id !== plan.plan_id
+            || view.run.plan_digest !== plan.plan_digest
+            || binding.binding.plan_id !== plan.plan_id) {
+            throw new Error("run ledger, plan, and binding cross-references disagree");
+        }
+        return {
+            plan_snapshot: canonicalizeJson(plan),
+            binding_snapshot: canonicalizeJson(binding),
+        };
+    }
+    catch (error) {
+        throw new PushDriverError("invalid_service_response", "service returned an invalid canonical plan or run binding", { cause: error });
+    }
+}
+function pinRun(view) {
+    const snapshots = validateRunPayloads(view);
+    return {
+        run_id: view.run.run_id,
+        plan_id: view.plan.plan_id,
+        plan_digest: view.plan.plan_digest,
+        binding_id: view.run.run_binding.binding_id,
+        max_concurrency: view.run.run_binding.binding.max_concurrency ?? 1,
+        ...snapshots,
+    };
+}
+function assertPinnedRun(pinned, view) {
+    const snapshots = validateRunPayloads(view);
+    if (view.run.run_id !== pinned.run_id
+        || view.plan.plan_id !== pinned.plan_id
+        || view.plan.plan_digest !== pinned.plan_digest
+        || view.run.plan_id !== pinned.plan_id
+        || view.run.plan_digest !== pinned.plan_digest
+        || view.run.run_binding.binding_id !== pinned.binding_id
+        || view.run.run_binding.binding.plan_id !== pinned.plan_id
+        || snapshots.plan_snapshot !== pinned.plan_snapshot
+        || snapshots.binding_snapshot !== pinned.binding_snapshot) {
+        throw new PushDriverError("invalid_service_response", "service response changed the pinned run, plan, or binding identity");
+    }
+    return view;
+}
+function requireWorkerNode(view, stepId) {
+    const matches = view.plan.plan.nodes.filter((node) => node.node_id === stepId);
+    if (matches.length !== 1 || matches[0]?.kind !== "worker") {
+        throw new PushDriverError("invalid_service_response", `acquired step ${stepId} is not exactly one pinned worker node`);
+    }
+    return matches[0];
+}
+function assertAttemptView(lease, workerSessionId, attached) {
+    const packet = attached.packet;
+    if (attached.lease.run_id !== lease.run_id
+        || attached.lease.step_id !== lease.step_id
+        || attached.lease.activation_id !== lease.activation_id
+        || attached.lease.attempt_id !== lease.attempt_id
+        || attached.lease.attempt_number !== lease.attempt_number
+        || attached.lease.claim_epoch !== lease.claim_epoch
+        || attached.lease.owner_id !== lease.owner_id
+        || attached.attempt.attempt_id !== lease.attempt_id
+        || attached.attempt.activation_id !== lease.activation_id
+        || attached.attempt.step_id !== lease.step_id
+        || attached.attempt.phase !== "running"
+        || attached.attempt.executor?.kind !== "worker"
+        || attached.attempt.executor.worker_session_id !== workerSessionId
+        || attached.attempt.started_at === null
+        || packet === null
+        || packet.run_id !== lease.run_id
+        || packet.step_id !== lease.step_id
+        || packet.attempt_id !== lease.attempt_id
+        || packet.attempt_number !== lease.attempt_number
+        || packet.worker_session_id !== workerSessionId) {
+        throw new PushDriverError("invalid_service_response", `attach response does not match attempt ${lease.attempt_id}`);
+    }
+    return packet;
+}
+function outputContractByName(node) {
+    return new Map(node.packet_template.output_contract.map((contract) => [
+        contract.name,
+        contract,
+    ]));
+}
+function referenceArtifactKind(contract) {
+    if (contract.artifact_kind !== "file"
+        && contract.artifact_kind !== "directory"
+        && contract.artifact_kind !== "git_commit") {
+        throw new PushDriverError("unsupported_output_contract", `reference output ${contract.name} uses unsupported artifact kind ${contract.artifact_kind}`);
+    }
+    return contract.artifact_kind;
+}
+function mapProviderOutput(output, contract) {
+    if (output.kind !== contract.kind) {
+        throw new PushDriverError("invalid_provider_output", `provider output ${output.name} kind does not match its pinned contract`);
+    }
+    if (output.kind === "text") {
+        return { name: contract.name, kind: "text", text: output.text };
+    }
+    if (output.kind === "structured") {
+        if (output.schema_ref !== contract.schema_ref) {
+            throw new PushDriverError("invalid_provider_output", `provider output ${output.name} schema does not match its pinned contract`);
+        }
+        canonicalizeJson(output.value);
+        return {
+            name: contract.name,
+            kind: "structured",
+            value: output.value,
+            schema_ref: contract.schema_ref,
+        };
+    }
+    const artifactKind = referenceArtifactKind(contract);
+    if (artifactKind === "git_commit") {
+        if (output.reference.kind !== "artifact") {
+            throw new PushDriverError("invalid_provider_output", `git_commit output ${output.name} must use an artifact reference`);
+        }
+        return {
+            name: contract.name,
+            kind: "reference",
+            artifact_kind: artifactKind,
+            source_path: output.reference.value,
+        };
+    }
+    if (output.reference.kind !== artifactKind) {
+        throw new PushDriverError("invalid_provider_output", `provider output ${output.name} reference kind does not match ${artifactKind}`);
+    }
+    if (contract.relative_path_template === null) {
+        throw new PushDriverError("unsupported_output_contract", `reference output ${contract.name} has no pinned source path`);
+    }
+    if (output.reference.value !== contract.relative_path_template) {
+        throw new PushDriverError("invalid_provider_output", `provider output ${output.name} path does not match its pinned contract`);
+    }
+    return {
+        name: contract.name,
+        kind: "reference",
+        artifact_kind: artifactKind,
+        source_path: contract.relative_path_template,
+    };
+}
+export function mapProviderOutputsToSubmission(node, outputs, requireRequiredOutputs) {
+    const contracts = outputContractByName(node);
+    const seen = new Set();
+    const mapped = [];
+    for (const output of outputs) {
+        if (seen.has(output.name)) {
+            throw new PushDriverError("invalid_provider_output", `provider returned duplicate output ${output.name}`);
+        }
+        seen.add(output.name);
+        const contract = contracts.get(output.name);
+        if (!contract) {
+            throw new PushDriverError("invalid_provider_output", `provider returned undeclared output ${output.name}`);
+        }
+        mapped.push(mapProviderOutput(output, contract));
+    }
+    if (requireRequiredOutputs) {
+        const missing = [...contracts.values()].find((contract) => (contract.required && !seen.has(contract.name)));
+        if (missing) {
+            throw new PushDriverError("invalid_provider_output", `provider omitted required output ${missing.name}`);
+        }
+    }
+    return mapped;
+}
+export function terminalOutcomeToCanonicalAttemptOutcome(process) {
+    switch (process.kind) {
+        case "success":
+            return {
+                kind: "worker_process",
+                status: "succeeded",
+                reason_code: null,
+                message: null,
+                process,
+            };
+        case "cancelled":
+            return {
+                kind: "worker_process",
+                status: "cancelled",
+                reason_code: "cancelled",
+                message: process.reason,
+                process,
+            };
+        case "malformed_result":
+        case "launch_error":
+            return {
+                kind: "worker_process",
+                status: "failed",
+                reason_code: process.kind,
+                message: process.error,
+                process,
+            };
+        case "non_zero_exit":
+        case "timeout":
+        case "signal":
+            return {
+                kind: "worker_process",
+                status: "failed",
+                reason_code: process.kind,
+                message: null,
+                process,
+            };
+    }
+}
+function stableSubmissionKey(lease, workerSessionId) {
+    const digest = createHash("sha256").update(canonicalizeJson({
+        run_id: lease.run_id,
+        activation_id: lease.activation_id,
+        attempt_id: lease.attempt_id,
+        worker_session_id: workerSessionId,
+    }), "utf8").digest("hex");
+    return `push-driver:${digest}`;
+}
+function digest(value) {
+    return `sha256:${createHash("sha256")
+        .update(canonicalizeJson(value), "utf8")
+        .digest("hex")}`;
+}
+function assertCallerAttestation(authority, request, attestation) {
+    const payload = attestation.payload;
+    if (authority.authority_id.length === 0
+        || payload === undefined
+        || payload.schema_version !== CALLER_ATTESTATION_SCHEMA_VERSION
+        || payload.authority_id !== authority.authority_id
+        || payload.agent_id.length === 0
+        || payload.session_id.length === 0
+        || payload.run_id !== request.run_id
+        || payload.step_id !== request.step_id
+        || payload.activation_id !== request.activation_id
+        || payload.attempt_id !== request.attempt_id
+        || payload.claim_epoch !== request.claim_epoch
+        || payload.worker_session_id !== request.worker_session_id
+        || payload.submission_key_digest !== request.submission_key_digest
+        || payload.execution_digest !== request.execution_digest
+        || attestation.attestation_ref.length === 0
+        || attestation.attestation_digest !== digest(payload)) {
+        throw new PushDriverError("invalid_execution_result", `caller attestation does not match attempt ${request.attempt_id}`);
+    }
+    return attestation;
+}
+function driverFailureKey(lease, workerSessionId) {
+    return `push-driver-failure:${createHash("sha256").update(canonicalizeJson({
+        run_id: lease.run_id,
+        attempt_id: lease.attempt_id,
+        worker_session_id: workerSessionId,
+    }), "utf8").digest("hex")}`;
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function assertExecutionIdentity(context, provider, execution) {
+    const result = execution.result;
+    if (execution.worker_session_id !== context.worker_session_id
+        || result.run_id !== context.lease.run_id
+        || result.step_id !== context.lease.step_id
+        || result.attempt_id !== context.lease.attempt_id
+        || result.worker_session_id !== context.worker_session_id
+        || result.provider !== provider.name
+        || result.started_at !== context.attached.attempt.started_at) {
+        throw new PushDriverError("invalid_execution_result", `worker execution identity does not match attempt ${context.lease.attempt_id}`);
+    }
+    const startedAt = Date.parse(result.started_at);
+    const completedAt = Date.parse(result.completed_at);
+    if (!Number.isFinite(startedAt)
+        || !Number.isFinite(completedAt)
+        || completedAt < startedAt) {
+        throw new PushDriverError("invalid_execution_result", `worker execution timestamps are invalid for attempt ${context.lease.attempt_id}`);
+    }
+    return result;
+}
+export class PushDriver {
+    capabilities;
+    service;
+    providerFactory;
+    executionAuthority;
+    callerAttestationAuthority;
+    lifecycleHooks;
+    diagnosticSink;
+    maxConcurrency;
+    heartbeatIntervalMs;
+    waitStrategy;
+    driverId;
+    createWorkerSessionId;
+    executeAttempt;
+    workerOptions;
+    sleep;
+    constructor(options) {
+        const maxConcurrency = options.max_concurrency ?? 1;
+        const heartbeatIntervalMs = options.heartbeat_interval_ms
+            ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+        const waitPollMs = options.wait_poll_ms ?? DEFAULT_WAIT_POLL_MS;
+        requirePositiveInteger("max_concurrency", maxConcurrency);
+        requirePositiveInteger("heartbeat_interval_ms", heartbeatIntervalMs);
+        requirePositiveInteger("wait_poll_ms", waitPollMs);
+        this.service = options.service;
+        this.providerFactory = options.provider_factory;
+        this.executionAuthority = options.execution_authority ?? null;
+        this.callerAttestationAuthority = options.caller_attestation_authority ?? null;
+        if (options.capabilities?.cancellation === true
+            && this.executionAuthority === null) {
+            throw new PushDriverError("invalid_configuration", "cancellation capability requires an execution authority binding factory");
+        }
+        if (options.capabilities?.caller_attestation === true
+            && this.callerAttestationAuthority === null) {
+            throw new PushDriverError("invalid_configuration", "caller attestation capability requires a trusted caller attestation authority");
+        }
+        this.lifecycleHooks = options.lifecycle_hooks ?? {};
+        this.diagnosticSink = options.diagnostic_sink ?? null;
+        this.maxConcurrency = maxConcurrency;
+        this.heartbeatIntervalMs = heartbeatIntervalMs;
+        this.waitStrategy = options.wait_strategy ?? defaultWaitStrategy(waitPollMs);
+        this.driverId = options.driver_id ?? `push-driver:${process.pid}:${randomUUID()}`;
+        if (this.driverId.length === 0) {
+            throw new PushDriverError("invalid_configuration", "driver_id must not be empty");
+        }
+        this.createWorkerSessionId = options.create_worker_session_id
+            ?? (() => randomUUID());
+        this.executeAttempt = options.execute_attempt ?? executeWorkerAttempt;
+        this.workerOptions = options.worker_options;
+        this.sleep = options.sleep ?? defaultSleep;
+        this.capabilities = Object.freeze({
+            ...DEFAULT_CAPABILITIES,
+            cancellation: this.executionAuthority !== null,
+            caller_attestation: this.callerAttestationAuthority !== null,
+            ...options.capabilities,
+        });
+    }
+    async emit(diagnostic) {
+        if (this.diagnosticSink === null)
+            return;
+        try {
+            await this.diagnosticSink(diagnostic);
+        }
+        catch {
+            // Diagnostics are observations and cannot become run authority.
+        }
+    }
+    async attemptError(context, lease, error) {
+        await this.emit({
+            type: "attempt_failed",
+            run_id: lease.run_id,
+            attempt_id: lease.attempt_id,
+            error,
+        });
+        try {
+            await this.lifecycleHooks.on_attempt_error?.(context, error);
+        }
+        catch {
+            // Preserve the original driving failure.
+        }
+    }
+    assertHeartbeatLease(lease, heartbeat) {
+        const heartbeatAt = Date.parse(heartbeat.heartbeat_at);
+        const expiresAt = Date.parse(heartbeat.claim_expires_at);
+        if (heartbeat.run_id !== lease.run_id
+            || heartbeat.step_id !== lease.step_id
+            || heartbeat.activation_id !== lease.activation_id
+            || heartbeat.attempt_id !== lease.attempt_id
+            || heartbeat.attempt_number !== lease.attempt_number
+            || heartbeat.claim_epoch !== lease.claim_epoch
+            || heartbeat.owner_id !== lease.owner_id
+            || heartbeat.execution_deadline_at !== lease.execution_deadline_at
+            || !Number.isFinite(heartbeatAt)
+            || !Number.isFinite(expiresAt)
+            || expiresAt <= heartbeatAt) {
+            throw new PushDriverError("invalid_service_response", `heartbeat response does not match attempt ${lease.attempt_id}`);
+        }
+    }
+    async startHeartbeat(lease) {
+        let stopped = false;
+        let failureError = NO_ATTEMPT_ERROR;
+        let timer;
+        let inFlight = Promise.resolve();
+        let rejectFailure;
+        const failure = new Promise((_resolve, reject) => {
+            rejectFailure = reject;
+        });
+        void failure.catch(() => undefined);
+        const heartbeat = async () => {
+            const response = await this.service.heartbeat({
+                lease_token: lease.claim_token,
+                owner_id: lease.owner_id,
+            });
+            this.assertHeartbeatLease(lease, response);
+            return response;
+        };
+        const schedule = (response) => {
+            if (stopped)
+                return;
+            const claimExpiresAt = Date.parse(response.claim_expires_at);
+            const executionDeadline = Date.parse(response.execution_deadline_at);
+            if (claimExpiresAt === executionDeadline)
+                return;
+            const leaseWindow = claimExpiresAt - Date.parse(response.heartbeat_at);
+            const delay = Math.max(1, Math.min(this.heartbeatIntervalMs, Math.floor(leaseWindow / 2)));
+            timer = setTimeout(() => {
+                inFlight = heartbeat().then((next) => schedule(next), async (error) => {
+                    failureError = error;
+                    await this.emit({
+                        type: "heartbeat_failed",
+                        run_id: lease.run_id,
+                        attempt_id: lease.attempt_id,
+                        error,
+                    });
+                    stopped = true;
+                    rejectFailure(error);
+                });
+            }, delay);
+            timer.unref?.();
+        };
+        schedule(await heartbeat());
+        return {
+            failure,
+            race: (operation) => (failureError === NO_ATTEMPT_ERROR
+                ? Promise.race([failure, operation])
+                : Promise.reject(failureError)),
+            stop: async () => {
+                stopped = true;
+                if (timer !== undefined)
+                    clearTimeout(timer);
+                await inFlight;
+            },
+        };
+    }
+    attachIsDefinitivelyFenced(view, lease, workerSessionId) {
+        const attempt = view.attempts.find((candidate) => candidate.attempt_id === lease.attempt_id);
+        const claims = view.claims
+            .filter((candidate) => candidate.attempt_id === lease.attempt_id)
+            .sort((left, right) => right.claim_epoch - left.claim_epoch);
+        const latestClaim = claims[0];
+        const attachedToOriginalWorker = attempt?.executor?.kind === "worker"
+            && attempt.executor.worker_session_id === workerSessionId;
+        if (attempt?.executor !== null && attempt?.executor !== undefined && !attachedToOriginalWorker) {
+            return true;
+        }
+        if (latestClaim !== undefined
+            && (latestClaim.claim_epoch !== lease.claim_epoch || latestClaim.owner_id !== lease.owner_id)
+            && !attachedToOriginalWorker) {
+            return true;
+        }
+        return (view.run.terminal !== null || view.run.status === "cancelling")
+            && !attachedToOriginalWorker;
+    }
+    async abandonPreparation(context, preparation, attachError) {
+        if (this.executionAuthority === null)
+            return;
+        try {
+            await this.executionAuthority.abandon(context, preparation);
+        }
+        catch (abandonError) {
+            throw new PushDriverError("authority_preparation_abandon_failed", `attempt ${context.lease.attempt_id} attach was fenced, but execution authority preparation cleanup failed: ${errorMessage(abandonError)}`, {
+                cause: new AggregateError([attachError, abandonError], "attempt attach fencing and authority preparation cleanup both failed"),
+            });
+        }
+    }
+    async reconcilePreparedAttach(request, preparationContext, preparation, originalError) {
+        const { lease, worker_session_id: workerSessionId } = preparationContext;
+        let inspected;
+        try {
+            inspected = await this.service.inspect(lease.run_id);
+        }
+        catch (inspectError) {
+            throw new PushDriverIndeterminateAttachError(lease.attempt_id, new AggregateError([originalError, inspectError], "attach and inspection failed"));
+        }
+        if (this.attachIsDefinitivelyFenced(inspected, lease, workerSessionId)) {
+            await this.abandonPreparation(preparationContext, preparation, originalError);
+            return { kind: "fenced", run: inspected };
+        }
+        try {
+            return { kind: "attached", value: await this.service.attachAttempt(request) };
+        }
+        catch (replayError) {
+            let replayInspection;
+            try {
+                replayInspection = await this.service.inspect(lease.run_id);
+            }
+            catch (inspectError) {
+                throw new PushDriverIndeterminateAttachError(lease.attempt_id, new AggregateError([originalError, replayError, inspectError], "attach replay and inspection failed"));
+            }
+            if (this.attachIsDefinitivelyFenced(replayInspection, lease, workerSessionId)) {
+                await this.abandonPreparation(preparationContext, preparation, replayError);
+                return { kind: "fenced", run: replayInspection };
+            }
+            throw new PushDriverIndeterminateAttachError(lease.attempt_id, new AggregateError([originalError, replayError], "attach disposition remains ambiguous"));
+        }
+    }
+    async driveAttempt(lease, run, workerSessionId, executionClaimEpoch, recoveryMode) {
+        let context = null;
+        let normalizedExecution = null;
+        let executionInvoked = false;
+        let processLaunchObserved = false;
+        let executionAuthority;
+        let launchAuthority = !recoveryMode;
+        let heartbeat = null;
+        try {
+            heartbeat = await this.startHeartbeat(lease);
+            const node = requireWorkerNode(run, lease.step_id);
+            await this.emit({
+                type: "attempt_acquired",
+                run_id: lease.run_id,
+                attempt_id: lease.attempt_id,
+                step_id: lease.step_id,
+            });
+            if (recoveryMode && this.executionAuthority === null) {
+                throw new PushDriverError("invalid_configuration", `attempt ${lease.attempt_id} cannot recover without execution authority`);
+            }
+            const authorityPreparationContext = {
+                run,
+                binding: run.run.run_binding.binding,
+                node,
+                lease,
+                worker_session_id: workerSessionId,
+                execution_claim_epoch: executionClaimEpoch,
+            };
+            const authorityPreparation = recoveryMode || this.executionAuthority === null
+                ? undefined
+                : await heartbeat.race(Promise.resolve(this.executionAuthority.prepare(authorityPreparationContext)));
+            const attachRequest = {
+                lease_token: lease.claim_token,
+                owner_id: lease.owner_id,
+                worker_identity: workerSessionId,
+                capabilities: this.capabilities,
+            };
+            const rawAttach = this.service.attachAttempt(attachRequest).then((value) => ({ kind: "attached", value }), (error) => ({ kind: "rejected", error }));
+            let attachOutcome;
+            try {
+                attachOutcome = await heartbeat.race(rawAttach);
+            }
+            catch (heartbeatError) {
+                await heartbeat.stop();
+                heartbeat = null;
+                const rawOutcome = await rawAttach;
+                if (recoveryMode || this.executionAuthority === null)
+                    throw heartbeatError;
+                const reconciliation = await this.reconcilePreparedAttach(attachRequest, authorityPreparationContext, authorityPreparation, rawOutcome.kind === "attached"
+                    ? heartbeatError
+                    : new AggregateError([heartbeatError, rawOutcome.error], "heartbeat and attach failed"));
+                if (reconciliation.kind === "fenced")
+                    return reconciliation.run;
+                assertAttemptView(lease, workerSessionId, reconciliation.value);
+                heartbeat = await this.startHeartbeat(lease);
+                attachOutcome = reconciliation;
+            }
+            if (attachOutcome.kind === "rejected") {
+                if (recoveryMode || this.executionAuthority === null)
+                    throw attachOutcome.error;
+                const reconciliation = await this.reconcilePreparedAttach(attachRequest, authorityPreparationContext, authorityPreparation, attachOutcome.error);
+                if (reconciliation.kind === "fenced")
+                    return reconciliation.run;
+                attachOutcome = reconciliation;
+            }
+            const attached = attachOutcome.value;
+            const packet = assertAttemptView(lease, workerSessionId, attached);
+            context = {
+                run,
+                binding: run.run.run_binding.binding,
+                node,
+                packet,
+                lease,
+                attached,
+                worker_session_id: workerSessionId,
+            };
+            await this.emit({
+                type: "attempt_attached",
+                run_id: lease.run_id,
+                attempt_id: lease.attempt_id,
+                worker_session_id: workerSessionId,
+            });
+            const startedAt = attached.attempt.started_at;
+            if (startedAt === null) {
+                throw new PushDriverError("invalid_service_response", `attempt ${lease.attempt_id} has no durable start time`);
+            }
+            if (this.executionAuthority !== null) {
+                const authorityContext = {
+                    ...context,
+                    started_at: startedAt,
+                    execution_claim_epoch: executionClaimEpoch,
+                };
+                if (recoveryMode) {
+                    const resolution = await heartbeat.race(Promise.resolve(this.executionAuthority.recover(authorityContext)));
+                    launchAuthority = resolution.kind === "launch";
+                    executionAuthority = resolution.binding;
+                }
+                else {
+                    executionAuthority = await heartbeat.race(Promise.resolve(this.executionAuthority.bind(authorityContext, authorityPreparation)));
+                }
+            }
+            await heartbeat.race(Promise.resolve(this.lifecycleHooks.before_execute?.(context)));
+            const provider = await heartbeat.race(Promise.resolve(this.providerFactory(context)));
+            if (!provider.name) {
+                throw new PushDriverError("invalid_configuration", "provider factory returned an unnamed provider");
+            }
+            const observedExecutionAuthority = executionAuthority === undefined
+                ? undefined
+                : {
+                    ...executionAuthority,
+                    processStarted: async (processId) => {
+                        processLaunchObserved = true;
+                        await executionAuthority.processStarted(processId);
+                    },
+                };
+            let execution;
+            let executionError = null;
+            const executionPromise = this.executeAttempt(packet, provider, {
+                ...this.workerOptions,
+                cancellationReason: null,
+                execution_authority: observedExecutionAuthority,
+                startedAt,
+            });
+            try {
+                executionInvoked = true;
+                execution = await heartbeat.race(executionPromise);
+            }
+            catch (error) {
+                executionError = error;
+            }
+            if (executionError !== null) {
+                void executionPromise.catch(() => undefined);
+                throw executionError;
+            }
+            if (execution === undefined) {
+                throw new PushDriverError("invalid_execution_result", `worker execution returned no result for attempt ${lease.attempt_id}`);
+            }
+            normalizedExecution = execution;
+            await this.lifecycleHooks.after_execute?.(context, execution);
+            const result = assertExecutionIdentity(context, provider, execution);
+            let outcome = terminalOutcomeToCanonicalAttemptOutcome(result.outcome);
+            let outputs;
+            try {
+                outputs = mapProviderOutputsToSubmission(node, result.outputs, outcome.kind === "worker_process" && outcome.status === "succeeded");
+            }
+            catch (error) {
+                outcome = {
+                    kind: "worker_process",
+                    status: "failed",
+                    reason_code: error instanceof PushDriverError
+                        ? error.code
+                        : "driver_postprocess_failed",
+                    message: errorMessage(error),
+                    process: result.outcome,
+                };
+                outputs = [];
+            }
+            const submissionKey = stableSubmissionKey(lease, workerSessionId);
+            const submissionExecution = {
+                executor: { kind: "worker", worker_session_id: workerSessionId },
+                started_at: result.started_at,
+                completed_at: result.completed_at,
+                outcome,
+                outputs,
+                workspace: {
+                    root: packet.workspace.working_directory,
+                    workspace_id: lease.attempt_id,
+                },
+                stdout_reference: null,
+                stderr_reference: null,
+            };
+            let evidenceRefs;
+            if (run.plan.plan.required_capabilities.caller_attestation) {
+                const authority = this.callerAttestationAuthority;
+                if (authority === null) {
+                    throw new PushDriverError("unsupported", "trusted caller attestation authority is unavailable");
+                }
+                const attestationRequest = {
+                    run_id: lease.run_id,
+                    step_id: lease.step_id,
+                    activation_id: lease.activation_id,
+                    attempt_id: lease.attempt_id,
+                    claim_epoch: lease.claim_epoch,
+                    worker_session_id: workerSessionId,
+                    provider_session_id: result.provider_session_id,
+                    submission_key_digest: digest({ submission_key: submissionKey }),
+                    execution_digest: digest(submissionExecution),
+                };
+                const attestation = await heartbeat.race(Promise.resolve(authority.attest(attestationRequest)));
+                evidenceRefs = [assertCallerAttestation(authority, attestationRequest, attestation)];
+            }
+            const response = await heartbeat.race(this.service.submitAttempt({
+                lease_token: lease.claim_token,
+                owner_id: lease.owner_id,
+                submission_key: submissionKey,
+                execution: submissionExecution,
+                ...(evidenceRefs === undefined ? {} : { evidence_refs: evidenceRefs }),
+            }));
+            if (response.replayed) {
+                await this.emit({
+                    type: "submission_replayed",
+                    run_id: lease.run_id,
+                    attempt_id: lease.attempt_id,
+                });
+            }
+            try {
+                await this.lifecycleHooks.after_submit?.(context, execution, response);
+            }
+            catch {
+                // Post-submit hooks are observations and cannot override a committed result.
+            }
+            return response.run;
+        }
+        catch (error) {
+            await this.attemptError(context, lease, error);
+            const canProvePrelaunch = launchAuthority && (!executionInvoked
+                || (executionAuthority !== undefined && !processLaunchObserved));
+            if (context !== null && normalizedExecution === null && canProvePrelaunch) {
+                try {
+                    return await this.service.settleDriverFailure({
+                        lease_token: lease.claim_token,
+                        owner_id: lease.owner_id,
+                        failure_key: driverFailureKey(lease, workerSessionId),
+                        reason_code: "driver_prelaunch_failed",
+                        message: errorMessage(error),
+                    });
+                }
+                catch (settlementError) {
+                    throw new PushDriverError("driver_failure_settlement_failed", `driver failed before launch, but canonical settlement was rejected: ${errorMessage(settlementError)}`, {
+                        cause: new AggregateError([error, settlementError], "driver failure and canonical settlement both failed"),
+                    });
+                }
+            }
+            throw error;
+        }
+        finally {
+            await heartbeat?.stop();
+        }
+    }
+    trackAttempt(lease, run, workerSessionId, executionClaimEpoch, recoveryMode) {
+        return this.driveAttempt(lease, run, workerSessionId, executionClaimEpoch, recoveryMode).then((nextRun) => ({
+            attempt_id: lease.attempt_id,
+            run: nextRun,
+            error: null,
+        }), (error) => ({
+            attempt_id: lease.attempt_id,
+            run: null,
+            error,
+        }));
+    }
+    async run(runId, options = {}) {
+        if (!runId) {
+            throw new PushDriverError("invalid_configuration", "run_id must not be empty");
+        }
+        if (options.initial_view !== undefined && options.resume === true) {
+            throw new PushDriverError("invalid_configuration", "initial_view and resume cannot be used together");
+        }
+        let current = options.initial_view
+            ?? (options.resume === true
+                ? await this.service.resume(runId)
+                : await this.service.inspect(runId));
+        if (current.run.run_id !== runId) {
+            throw new PushDriverError("invalid_service_response", "service returned a different run_id");
+        }
+        const pinned = pinRun(current);
+        if (pinned.max_concurrency !== this.maxConcurrency) {
+            throw new PushDriverError("invalid_configuration", `driver max_concurrency ${this.maxConcurrency} does not match immutable run binding ${pinned.max_concurrency}`);
+        }
+        current = assertPinnedRun(pinned, current);
+        assertRequiredCapabilities(current.plan.plan.required_capabilities, this.capabilities);
+        const active = new Map();
+        const activeDeadlines = new Map();
+        const launchedAttempts = new Set();
+        const workerSessions = new Set();
+        let firstAttemptError = NO_ATTEMPT_ERROR;
+        let cancellationRequested = false;
+        const abandonActiveWaits = () => {
+            for (const promise of active.values()) {
+                void promise.then(() => undefined, () => undefined);
+            }
+            active.clear();
+            activeDeadlines.clear();
+        };
+        const cancelAndDrainAfterDriverFailure = async (error) => {
+            if (active.size === 0)
+                throw error;
+            if (!cancellationRequested) {
+                cancellationRequested = true;
+                current = assertPinnedRun(pinned, await this.service.cancel(runId, `push driver failed: ${errorMessage(error)}`));
+            }
+            const latestDeadline = Math.max(Date.now(), ...activeDeadlines.values());
+            abandonActiveWaits();
+            while (current.run.terminal === null) {
+                current = assertPinnedRun(pinned, await this.service.inspect(runId));
+                if (current.run.terminal !== null)
+                    break;
+                if (Date.now() >= latestDeadline)
+                    throw error;
+                const decision = await this.waitStrategy({
+                    reason_code: "run_cancelling",
+                    run: current,
+                    active_attempts: 0,
+                });
+                if (decision.action === "stop")
+                    throw error;
+                const remaining = latestDeadline - Date.now();
+                await this.sleep(Math.min(decision.delay_ms, remaining));
+            }
+            throw error;
+        };
+        for (;;) {
+            if (current.run.terminal !== null) {
+                if (active.size > 0) {
+                    const siblings = [...active.values()];
+                    active.clear();
+                    activeDeadlines.clear();
+                    await Promise.allSettled(siblings);
+                }
+                return current;
+            }
+            let waiting = null;
+            while (active.size < this.maxConcurrency
+                && firstAttemptError === NO_ATTEMPT_ERROR) {
+                try {
+                    const acquired = await this.service.acquireAttempt({
+                        run_id: runId,
+                        driver_id: this.driverId,
+                        recover_running: options.resume === true,
+                    });
+                    current = assertPinnedRun(pinned, current);
+                    if (acquired.kind === "terminal") {
+                        current = assertPinnedRun(pinned, acquired.run);
+                        break;
+                    }
+                    if (acquired.kind === "wait") {
+                        current = assertPinnedRun(pinned, acquired.run);
+                        waiting = acquired;
+                        break;
+                    }
+                    const lease = acquired.lease;
+                    if (lease.run_id !== runId || lease.owner_id !== this.driverId) {
+                        throw new PushDriverError("invalid_service_response", "acquired lease does not belong to this driver and run");
+                    }
+                    requireNonEmpty("attempt_id", lease.attempt_id);
+                    if (launchedAttempts.has(lease.attempt_id)) {
+                        throw new PushDriverError("attempt_relaunch", `service attempted to relaunch ${lease.attempt_id}`);
+                    }
+                    const executionDeadline = Date.parse(lease.execution_deadline_at);
+                    if (!Number.isFinite(executionDeadline)) {
+                        throw new PushDriverError("invalid_service_response", `attempt ${lease.attempt_id} has an invalid execution deadline`);
+                    }
+                    const workerSessionId = acquired.recovery?.worker_session_id
+                        ?? this.createWorkerSessionId();
+                    requireNonEmpty("worker_session_id", workerSessionId);
+                    if (workerSessions.has(workerSessionId)) {
+                        throw new PushDriverError("invalid_configuration", `worker session factory reused ${workerSessionId}`);
+                    }
+                    launchedAttempts.add(lease.attempt_id);
+                    workerSessions.add(workerSessionId);
+                    const tracked = this.trackAttempt(lease, current, workerSessionId, acquired.recovery?.execution_claim_epoch ?? lease.claim_epoch, acquired.recovery !== null).then((completion) => {
+                        if (completion.error !== null
+                            && firstAttemptError === NO_ATTEMPT_ERROR) {
+                            firstAttemptError = completion.error;
+                        }
+                        return completion;
+                    });
+                    active.set(lease.attempt_id, tracked);
+                    activeDeadlines.set(lease.attempt_id, executionDeadline);
+                }
+                catch (error) {
+                    await cancelAndDrainAfterDriverFailure(error);
+                }
+            }
+            if (current.run.terminal !== null)
+                continue;
+            if (active.size > 0) {
+                const completion = await Promise.race(active.values());
+                const completedDeadline = activeDeadlines.get(completion.attempt_id);
+                active.delete(completion.attempt_id);
+                activeDeadlines.delete(completion.attempt_id);
+                if (completion.error !== null) {
+                    const attemptError = firstAttemptError === NO_ATTEMPT_ERROR
+                        ? completion.error
+                        : firstAttemptError;
+                    if (attemptError instanceof PushDriverIndeterminateAttachError) {
+                        const siblings = [...active.values()];
+                        active.clear();
+                        activeDeadlines.clear();
+                        await Promise.allSettled(siblings);
+                        throw attemptError;
+                    }
+                    if (!cancellationRequested) {
+                        cancellationRequested = true;
+                        current = assertPinnedRun(pinned, await this.service.cancel(runId, `push driver attempt failed: ${errorMessage(attemptError)}`));
+                        if (current.run.terminal !== null) {
+                            abandonActiveWaits();
+                            continue;
+                        }
+                    }
+                    const latestDeadline = Math.max(Date.now(), completedDeadline ?? Number.NEGATIVE_INFINITY, ...activeDeadlines.values());
+                    abandonActiveWaits();
+                    for (;;) {
+                        current = assertPinnedRun(pinned, await this.service.inspect(runId));
+                        if (current.run.terminal !== null)
+                            break;
+                        if (Date.now() >= latestDeadline)
+                            throw attemptError;
+                        const decision = await this.waitStrategy({
+                            reason_code: "run_cancelling",
+                            run: current,
+                            active_attempts: 0,
+                        });
+                        if (decision.action === "stop")
+                            throw attemptError;
+                        const remaining = latestDeadline - Date.now();
+                        await this.sleep(Math.min(decision.delay_ms, remaining));
+                    }
+                    continue;
+                }
+                if (completion.run === null) {
+                    throw new PushDriverError("invalid_service_response", `attempt ${completion.attempt_id} completed without a run view`);
+                }
+                current = assertPinnedRun(pinned, completion.run);
+                continue;
+            }
+            if (waiting === null) {
+                throw new PushDriverError("invalid_service_response", "driver has neither an active attempt nor a typed wait response");
+            }
+            const decision = await this.waitStrategy({
+                reason_code: waiting.reason_code,
+                run: current,
+                active_attempts: 0,
+            });
+            if (decision.action === "stop") {
+                await this.emit({
+                    type: "wait",
+                    run_id: runId,
+                    reason_code: waiting.reason_code,
+                    delay_ms: null,
+                });
+                throw new PushDriverError("wait_stopped", decision.reason ?? `wait strategy stopped on ${waiting.reason_code}`);
+            }
+            if (!Number.isSafeInteger(decision.delay_ms) || decision.delay_ms < 0) {
+                throw new PushDriverError("invalid_configuration", "wait strategy delay_ms must be a non-negative safe integer");
+            }
+            await this.emit({
+                type: "wait",
+                run_id: runId,
+                reason_code: waiting.reason_code,
+                delay_ms: decision.delay_ms,
+            });
+            await this.sleep(decision.delay_ms);
+        }
+    }
+}