intentdna 1.8.6 → 1.8.7

package/dist/runtime/run-controller.js

@@ -0,0 +1,1108 @@
+import { randomUUID } from "node:crypto";
+import { HandoffResolver, } from "./handoff-resolver.js";
+import { ImmutableResultStore } from "./result-store.js";
+import { RunStoreError, } from "./run-store.js";
+import { executeWorkerAttempt, } from "./worker-executor.js";
+export class RunControllerError extends Error {
+    code;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = "RunControllerError";
+        this.code = code;
+    }
+}
+const DEFAULT_SCHEDULER_POLL_MS = 100;
+const TERMINAL_TASK_STATES = new Set([
+    "succeeded",
+    "failed",
+    "skipped",
+    "cancelled",
+]);
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isVersionConflict(error) {
+    return error instanceof RunStoreError && error.code === "version_conflict";
+}
+function terminalTask(task) {
+    return TERMINAL_TASK_STATES.has(task.state);
+}
+function assertPositiveInteger(name, value) {
+    if (!Number.isSafeInteger(value) || value <= 0) {
+        throw new RunControllerError("invalid_configuration", `${name} must be a positive safe integer`);
+    }
+}
+function taskByStep(snapshot, stepId) {
+    return snapshot.tasks.find((task) => task.step_id === stepId) ?? null;
+}
+function attemptById(snapshot, attemptId) {
+    if (attemptId === null)
+        return null;
+    return snapshot.attempts.find((attempt) => attempt.attempt_id === attemptId) ?? null;
+}
+function claimByAttempt(snapshot, attemptId) {
+    return snapshot.claims.find((claim) => claim.attempt_id === attemptId) ?? null;
+}
+function iso(date) {
+    return date.toISOString();
+}
+function retryDelayMs(policy, completedAttemptNumber) {
+    if (policy.backoff === "none")
+        return 0;
+    const multiplier = policy.backoff === "linear"
+        ? completedAttemptNumber
+        : 2 ** Math.max(0, completedAttemptNumber - 1);
+    const delay = policy.initial_delay_ms * multiplier;
+    return policy.max_delay_ms === null
+        ? delay
+        : Math.min(delay, policy.max_delay_ms);
+}
+function retryAllowed(policy, attempt, outcome) {
+    return (attempt.attempt_number < policy.max_attempts
+        && policy.retryable_outcomes.includes(outcome.kind));
+}
+function eventPayload(value) {
+    return value;
+}
+export class RunController {
+    store;
+    resolver;
+    definitions;
+    definitionByStep;
+    providerForStep;
+    maxConcurrency;
+    leaseDurationMs;
+    leaseHeartbeatMs;
+    schedulerPollMs;
+    ownerId;
+    reconcileAttempt;
+    executeAttempt;
+    workerOptions;
+    now;
+    createAttemptId;
+    createWorkerSessionId;
+    createClaimToken;
+    createResultId;
+    createEventId;
+    active = new Map();
+    constructor(options) {
+        assertPositiveInteger("max_concurrency", options.max_concurrency);
+        assertPositiveInteger("lease_duration_ms", options.lease_duration_ms);
+        const heartbeat = options.lease_heartbeat_ms
+            ?? Math.max(1, Math.floor(options.lease_duration_ms / 3));
+        assertPositiveInteger("lease_heartbeat_ms", heartbeat);
+        if (heartbeat >= options.lease_duration_ms) {
+            throw new RunControllerError("invalid_configuration", "lease_heartbeat_ms must be less than lease_duration_ms");
+        }
+        const poll = options.scheduler_poll_ms ?? DEFAULT_SCHEDULER_POLL_MS;
+        assertPositiveInteger("scheduler_poll_ms", poll);
+        this.store = options.run_store;
+        this.resolver = options.handoff_resolver
+            ?? new HandoffResolver(new ImmutableResultStore(options.run_store));
+        this.definitions = [...options.steps];
+        this.definitionByStep = this.validateDefinitions(this.definitions);
+        this.providerForStep = options.provider_for_step;
+        this.maxConcurrency = options.max_concurrency;
+        this.leaseDurationMs = options.lease_duration_ms;
+        this.leaseHeartbeatMs = heartbeat;
+        this.schedulerPollMs = poll;
+        this.ownerId = options.owner_id ?? `controller:${process.pid}:${randomUUID()}`;
+        this.reconcileAttempt = options.reconcile_attempt
+            ?? (async (attempt) => this.active.has(attempt.attempt_id) ? "live" : "unknown");
+        this.executeAttempt = options.execute_attempt ?? executeWorkerAttempt;
+        this.workerOptions = options.worker_options;
+        this.now = options.now ?? (() => new Date());
+        this.createAttemptId = options.create_attempt_id
+            ?? (() => randomUUID());
+        this.createWorkerSessionId = options.create_worker_session_id
+            ?? (() => randomUUID());
+        this.createClaimToken = options.create_claim_token
+            ?? (() => randomUUID());
+        this.createResultId = options.create_result_id
+            ?? (() => randomUUID());
+        this.createEventId = options.create_event_id
+            ?? (() => randomUUID());
+    }
+    validateDefinitions(definitions) {
+        const byStep = new Map();
+        for (const definition of definitions) {
+            if (!definition.step_id || byStep.has(definition.step_id)) {
+                throw new RunControllerError("invalid_definition", `duplicate or empty step definition ${definition.step_id}`);
+            }
+            if (!Number.isSafeInteger(definition.retry_policy.max_attempts)
+                || definition.retry_policy.max_attempts < 1
+                || !Number.isSafeInteger(definition.retry_policy.initial_delay_ms)
+                || definition.retry_policy.initial_delay_ms < 0
+                || (definition.retry_policy.max_delay_ms !== null
+                    && (!Number.isSafeInteger(definition.retry_policy.max_delay_ms)
+                        || definition.retry_policy.max_delay_ms < 0))) {
+                throw new RunControllerError("invalid_definition", `step ${definition.step_id} has an invalid retry policy`);
+            }
+            if (new Set(definition.dependencies).size !== definition.dependencies.length) {
+                throw new RunControllerError("invalid_definition", `step ${definition.step_id} has duplicate dependencies`);
+            }
+            byStep.set(definition.step_id, definition);
+        }
+        for (const definition of definitions) {
+            for (const dependency of definition.dependencies) {
+                if (dependency === definition.step_id || !byStep.has(dependency)) {
+                    throw new RunControllerError("invalid_definition", `step ${definition.step_id} has invalid dependency ${dependency}`);
+                }
+            }
+        }
+        return byStep;
+    }
+    events(snapshot, inputs) {
+        return inputs.map((input, index) => ({
+            ...input,
+            event_id: this.createEventId(),
+            sequence: snapshot.events.length + index + 1,
+        }));
+    }
+    async createRun(input) {
+        const createdAt = input.run.created_at;
+        const tasks = this.definitions.map((definition) => ({
+            run_id: input.run.run_id,
+            step_id: definition.step_id,
+            state: definition.initially_skipped
+                ? "skipped"
+                : definition.dependencies.length === 0
+                    ? "ready"
+                    : "dependency_blocked",
+            attempt_count: 0,
+            current_attempt_id: null,
+            committed_result_id: null,
+            created_at: createdAt,
+            updated_at: createdAt,
+        }));
+        let snapshot = await this.store.createRun({ run: input.run, tasks });
+        const occurredAt = iso(this.now());
+        const update = await this.store.updateRun({
+            run_id: input.run.run_id,
+            expected_record_version: snapshot.record_version,
+            events: this.events(snapshot, [{
+                    run_id: input.run.run_id,
+                    step_id: null,
+                    attempt_id: null,
+                    worker_session_id: null,
+                    result_id: null,
+                    idempotency_key: `run:${input.run.run_id}:created`,
+                    type: "run_created",
+                    occurred_at: occurredAt,
+                    payload: eventPayload({ plan_digest: input.run.plan_digest }),
+                }]),
+        });
+        snapshot = update.snapshot;
+        return snapshot;
+    }
+    async inspect(runId) {
+        return this.store.requireRun(runId);
+    }
+    dependenciesFor(definition, snapshot) {
+        return definition.dependencies.map((stepId) => {
+            const task = taskByStep(snapshot, stepId);
+            if (!task) {
+                throw new RunControllerError("invalid_definition", `run ${snapshot.run.run_id} has no task for dependency ${stepId}`);
+            }
+            return task;
+        });
+    }
+    /**
+     * A dependency is satisfied only by a succeeded task whose referenced result
+     * is present in the immutable committed-result collection.
+     */
+    async refreshDependencyReadiness(runId) {
+        while (true) {
+            const snapshot = await this.store.requireRun(runId);
+            if (snapshot.run.status !== "active")
+                return snapshot;
+            const now = iso(this.now());
+            const changed = [];
+            const eventInputs = [];
+            for (const task of snapshot.tasks) {
+                if (task.state !== "dependency_blocked")
+                    continue;
+                const definition = this.definitionByStep.get(task.step_id);
+                if (!definition) {
+                    throw new RunControllerError("invalid_definition", `run ${runId} contains unknown step ${task.step_id}`);
+                }
+                const dependencies = this.dependenciesFor(definition, snapshot);
+                const failedDependency = dependencies.some((dependency) => terminalTask(dependency) && dependency.state !== "succeeded");
+                const ready = dependencies.every((dependency) => (dependency.state === "succeeded"
+                    && dependency.committed_result_id !== null
+                    && snapshot.results.some((result) => result.result_id === dependency.committed_result_id)));
+                if (!failedDependency && !ready)
+                    continue;
+                const state = failedDependency ? "skipped" : "ready";
+                changed.push({ ...task, state, updated_at: now });
+                eventInputs.push({
+                    run_id: runId,
+                    step_id: task.step_id,
+                    attempt_id: null,
+                    worker_session_id: null,
+                    result_id: null,
+                    idempotency_key: `task:${task.step_id}:dependency:${state}`,
+                    type: "task_state_changed",
+                    occurred_at: now,
+                    payload: eventPayload({
+                        from: task.state,
+                        to: state,
+                        reason: failedDependency
+                            ? "dependency_terminal_without_success"
+                            : "dependencies_committed",
+                    }),
+                });
+            }
+            if (changed.length === 0)
+                return snapshot;
+            try {
+                return (await this.store.updateRun({
+                    run_id: runId,
+                    expected_record_version: snapshot.record_version,
+                    tasks: changed,
+                    events: this.events(snapshot, eventInputs),
+                })).snapshot;
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    readyAt(definition, task, snapshot) {
+        const prior = attemptById(snapshot, task.current_attempt_id);
+        if (!prior || prior.phase !== "terminal" || prior.completed_at === null) {
+            return 0;
+        }
+        return (Date.parse(prior.completed_at)
+            + retryDelayMs(definition.retry_policy, prior.attempt_number));
+    }
+    async claimNextReady(runId) {
+        while (true) {
+            const snapshot = await this.refreshDependencyReadiness(runId);
+            if (snapshot.run.status !== "active")
+                return null;
+            const activeCount = snapshot.tasks.filter((task) => task.state === "claimed" || task.state === "running").length;
+            if (activeCount >= this.maxConcurrency)
+                return null;
+            const nowDate = this.now();
+            const candidate = this.definitions.find((definition) => {
+                const task = taskByStep(snapshot, definition.step_id);
+                return (task?.state === "ready"
+                    && task.committed_result_id === null
+                    && this.readyAt(definition, task, snapshot) <= nowDate.getTime());
+            });
+            if (!candidate)
+                return null;
+            const task = taskByStep(snapshot, candidate.step_id);
+            if (!task) {
+                throw new RunControllerError("invalid_definition", `run ${runId} has no task for step ${candidate.step_id}`);
+            }
+            const attemptId = this.createAttemptId();
+            const workerSessionId = this.createWorkerSessionId();
+            const claimToken = this.createClaimToken();
+            const now = iso(nowDate);
+            const expiresAt = iso(new Date(nowDate.getTime() + this.leaseDurationMs));
+            const attempt = {
+                run_id: runId,
+                step_id: candidate.step_id,
+                attempt_id: attemptId,
+                worker_session_id: workerSessionId,
+                attempt_number: task.attempt_count + 1,
+                phase: "claimed",
+                claim_token: claimToken,
+                process_id: null,
+                started_at: null,
+                completed_at: null,
+                terminal_outcome: null,
+            };
+            const claim = {
+                run_id: runId,
+                step_id: candidate.step_id,
+                attempt_id: attemptId,
+                worker_session_id: workerSessionId,
+                claim_token: claimToken,
+                owner_id: this.ownerId,
+                claimed_record_version: snapshot.record_version + 1,
+                lease: {
+                    acquired_at: now,
+                    heartbeat_at: now,
+                    expires_at: expiresAt,
+                },
+            };
+            const updatedTask = {
+                ...task,
+                state: "claimed",
+                attempt_count: attempt.attempt_number,
+                current_attempt_id: attemptId,
+                updated_at: now,
+            };
+            const events = this.events(snapshot, [
+                {
+                    run_id: runId,
+                    step_id: task.step_id,
+                    attempt_id: attemptId,
+                    worker_session_id: workerSessionId,
+                    result_id: null,
+                    idempotency_key: `claim:${claimToken}:acquired`,
+                    type: "task_claimed",
+                    occurred_at: now,
+                    payload: eventPayload({
+                        claim_token: claimToken,
+                        owner_id: this.ownerId,
+                        expires_at: expiresAt,
+                    }),
+                },
+                {
+                    run_id: runId,
+                    step_id: task.step_id,
+                    attempt_id: attemptId,
+                    worker_session_id: workerSessionId,
+                    result_id: null,
+                    idempotency_key: `task:${task.step_id}:attempt:${attemptId}:claimed`,
+                    type: "task_state_changed",
+                    occurred_at: now,
+                    payload: eventPayload({ from: "ready", to: "claimed" }),
+                },
+            ]);
+            try {
+                const update = await this.store.updateRun({
+                    run_id: runId,
+                    expected_record_version: snapshot.record_version,
+                    tasks: [updatedTask],
+                    attempts: [attempt],
+                    claims: [claim],
+                    events,
+                });
+                return {
+                    definition: candidate,
+                    attempt,
+                    claim,
+                    snapshot: update.snapshot,
+                };
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    assertCurrentClaim(snapshot, claim, requireUnexpired) {
+        const task = taskByStep(snapshot, claim.step_id);
+        const attempt = attemptById(snapshot, claim.attempt_id);
+        const storedClaim = claimByAttempt(snapshot, claim.attempt_id);
+        if (!task
+            || !attempt
+            || !storedClaim
+            || task.current_attempt_id !== claim.attempt_id
+            || attempt.claim_token !== claim.claim_token
+            || storedClaim.claim_token !== claim.claim_token
+            || storedClaim.owner_id !== claim.owner_id) {
+            throw new RunControllerError("claim_lost", `claim ${claim.claim_token} no longer owns step ${claim.step_id}`);
+        }
+        if (requireUnexpired
+            && Date.parse(storedClaim.lease.expires_at) <= this.now().getTime()) {
+            throw new RunControllerError("claim_lost", `claim ${claim.claim_token} expired before mutation`);
+        }
+        return { task, attempt };
+    }
+    async renewClaim(claim) {
+        while (true) {
+            const snapshot = await this.store.requireRun(claim.run_id);
+            const { attempt } = this.assertCurrentClaim(snapshot, claim, true);
+            if (snapshot.run.status === "cancelling") {
+                throw new RunControllerError("run_cancelling", snapshot.run.cancellation?.reason ?? "run cancellation requested");
+            }
+            if (attempt.phase === "terminal") {
+                throw new RunControllerError("claim_lost", `terminal attempt ${attempt.attempt_id} cannot renew its claim`);
+            }
+            const storedClaim = claimByAttempt(snapshot, claim.attempt_id);
+            const nowDate = this.now();
+            const renewed = {
+                ...storedClaim,
+                lease: {
+                    ...storedClaim.lease,
+                    heartbeat_at: iso(nowDate),
+                    expires_at: iso(new Date(nowDate.getTime() + this.leaseDurationMs)),
+                },
+            };
+            const event = this.events(snapshot, [{
+                    run_id: claim.run_id,
+                    step_id: claim.step_id,
+                    attempt_id: claim.attempt_id,
+                    worker_session_id: claim.worker_session_id,
+                    result_id: null,
+                    idempotency_key: `claim:${claim.claim_token}:heartbeat:${renewed.lease.heartbeat_at}`,
+                    type: "task_lease_renewed",
+                    occurred_at: renewed.lease.heartbeat_at,
+                    payload: eventPayload({ expires_at: renewed.lease.expires_at }),
+                }]);
+            try {
+                await this.store.updateRun({
+                    run_id: claim.run_id,
+                    expected_record_version: snapshot.record_version,
+                    claims: [renewed],
+                    events: event,
+                });
+                return renewed;
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    async markRunning(claim) {
+        while (true) {
+            const snapshot = await this.store.requireRun(claim.run_id);
+            const { task, attempt } = this.assertCurrentClaim(snapshot, claim, true);
+            if (snapshot.run.status === "cancelling") {
+                throw new RunControllerError("run_cancelling", snapshot.run.cancellation?.reason ?? "run cancellation requested");
+            }
+            if (task.state === "running" && attempt.phase === "running")
+                return;
+            if (task.state !== "claimed" || attempt.phase !== "claimed") {
+                throw new RunControllerError("claim_lost", `attempt ${attempt.attempt_id} cannot enter running from ${attempt.phase}`);
+            }
+            const now = iso(this.now());
+            const events = this.events(snapshot, [
+                {
+                    run_id: claim.run_id,
+                    step_id: claim.step_id,
+                    attempt_id: claim.attempt_id,
+                    worker_session_id: claim.worker_session_id,
+                    result_id: null,
+                    idempotency_key: `attempt:${claim.attempt_id}:started`,
+                    type: "attempt_started",
+                    occurred_at: now,
+                    payload: eventPayload({ claim_token: claim.claim_token }),
+                },
+                {
+                    run_id: claim.run_id,
+                    step_id: claim.step_id,
+                    attempt_id: claim.attempt_id,
+                    worker_session_id: claim.worker_session_id,
+                    result_id: null,
+                    idempotency_key: `task:${claim.step_id}:attempt:${claim.attempt_id}:running`,
+                    type: "task_state_changed",
+                    occurred_at: now,
+                    payload: eventPayload({ from: "claimed", to: "running" }),
+                },
+            ]);
+            try {
+                await this.store.updateRun({
+                    run_id: claim.run_id,
+                    expected_record_version: snapshot.record_version,
+                    tasks: [{ ...task, state: "running", updated_at: now }],
+                    attempts: [{ ...attempt, phase: "running", started_at: now }],
+                    events,
+                });
+                return;
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    handoffRequest(declaration, snapshot) {
+        if (declaration.mode === "none") {
+            return declaration;
+        }
+        const sourceTask = taskByStep(snapshot, declaration.source.step_id);
+        if (!sourceTask?.committed_result_id) {
+            if (!declaration.required) {
+                return {
+                    binding_id: declaration.binding_id,
+                    input_name: declaration.input_name,
+                    required: false,
+                    description: declaration.description,
+                    mode: "none",
+                    reason: "optional source task has no committed result",
+                };
+            }
+            throw new RunControllerError("invalid_packet", `handoff ${declaration.binding_id} has no committed source result`);
+        }
+        const source = {
+            result_id: sourceTask.committed_result_id,
+            step_id: declaration.source.step_id,
+            output_name: declaration.source.output_name,
+        };
+        if (declaration.mode === "reference") {
+            return { ...declaration, source };
+        }
+        if (declaration.mode === "quote") {
+            return { ...declaration, source };
+        }
+        return { ...declaration, source };
+    }
+    async persistResolvedHandoffs(claim, bindings) {
+        const occurredAt = iso(this.now());
+        while (true) {
+            const snapshot = await this.store.requireRun(claim.run_id);
+            const { task, attempt } = this.assertCurrentClaim(snapshot, claim, true);
+            if (task.state !== "running" || attempt.phase !== "running") {
+                throw new RunControllerError("claim_lost", `attempt ${claim.attempt_id} is no longer running`);
+            }
+            const existingKeys = new Set(snapshot.events.map((event) => event.idempotency_key));
+            const missing = bindings.filter((binding) => !existingKeys.has(`attempt:${claim.attempt_id}:handoff:${binding.binding_id}`));
+            const ordered = [
+                ...missing,
+                ...bindings.filter((binding) => !missing.includes(binding)),
+            ];
+            const missingCount = missing.length;
+            const events = ordered.map((binding, index) => ({
+                event_id: this.createEventId(),
+                run_id: claim.run_id,
+                step_id: claim.step_id,
+                attempt_id: claim.attempt_id,
+                worker_session_id: claim.worker_session_id,
+                result_id: binding.mode === "none" ? null : binding.source.result_id,
+                sequence: index < missingCount
+                    ? snapshot.events.length + index + 1
+                    : snapshot.events.length,
+                idempotency_key: `attempt:${claim.attempt_id}:handoff:${binding.binding_id}`,
+                type: "handoff_resolved",
+                occurred_at: occurredAt,
+                payload: { ...binding },
+            }));
+            try {
+                await this.store.updateRun({
+                    run_id: claim.run_id,
+                    expected_record_version: snapshot.record_version,
+                    events,
+                });
+                return;
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    async materializePacket(claimed) {
+        const snapshot = await this.store.requireRun(claimed.claim.run_id);
+        this.assertCurrentClaim(snapshot, claimed.claim, true);
+        const requests = claimed.definition.handoffs.map((declaration) => this.handoffRequest(declaration, snapshot));
+        const bindings = await this.resolver.resolveAll(claimed.claim.run_id, requests);
+        const createdAt = iso(this.now());
+        const packet = claimed.definition.materialize_packet({
+            run_id: claimed.claim.run_id,
+            step_id: claimed.claim.step_id,
+            attempt_id: claimed.claim.attempt_id,
+            worker_session_id: claimed.claim.worker_session_id,
+            attempt_number: claimed.attempt.attempt_number,
+            input_bindings: bindings,
+            retry_policy: claimed.definition.retry_policy,
+            created_at: createdAt,
+        });
+        if (packet.run_id !== claimed.claim.run_id
+            || packet.step_id !== claimed.claim.step_id
+            || packet.attempt_id !== claimed.claim.attempt_id
+            || packet.worker_session_id !== claimed.claim.worker_session_id
+            || packet.attempt_number !== claimed.attempt.attempt_number
+            || JSON.stringify(packet.input_bindings) !== JSON.stringify(bindings)
+            || JSON.stringify(packet.execution.retry)
+                !== JSON.stringify(claimed.definition.retry_policy)) {
+            throw new RunControllerError("invalid_packet", `materialized packet for ${claimed.claim.step_id} changed Controller-owned identities or inputs`);
+        }
+        await this.persistResolvedHandoffs(claimed.claim, bindings);
+        return packet;
+    }
+    async finalizeAttempt(claim, execution) {
+        while (true) {
+            const snapshot = await this.store.requireRun(claim.run_id);
+            const { task, attempt } = this.assertCurrentClaim(snapshot, claim, true);
+            if (attempt.phase === "terminal")
+                return snapshot;
+            if (execution.worker_session_id !== claim.worker_session_id
+                || execution.result.run_id !== claim.run_id
+                || execution.result.step_id !== claim.step_id
+                || execution.result.attempt_id !== claim.attempt_id
+                || execution.result.worker_session_id !== claim.worker_session_id) {
+                throw new RunControllerError("invalid_packet", `worker result does not match claimed attempt ${claim.attempt_id}`);
+            }
+            const now = iso(this.now());
+            const runCancelling = snapshot.run.status === "cancelling";
+            const canRetry = (!runCancelling
+                && retryAllowed(this.definitionByStep.get(claim.step_id).retry_policy, attempt, execution.result.outcome));
+            const resultId = execution.result.outcome.kind === "success"
+                ? this.createResultId()
+                : null;
+            const result = resultId === null
+                ? null
+                : {
+                    ...execution.result,
+                    result_id: resultId,
+                    commit_status: "committed",
+                    committed_at: now,
+                };
+            const nextState = result
+                ? "succeeded"
+                : runCancelling
+                    ? "cancelled"
+                    : canRetry
+                        ? "ready"
+                        : "failed";
+            const terminalAttempt = {
+                ...attempt,
+                phase: "terminal",
+                completed_at: execution.result.completed_at,
+                terminal_outcome: execution.result.outcome,
+            };
+            const updatedTask = {
+                ...task,
+                state: nextState,
+                committed_result_id: resultId,
+                updated_at: now,
+            };
+            const eventInputs = [
+                {
+                    run_id: claim.run_id,
+                    step_id: claim.step_id,
+                    attempt_id: claim.attempt_id,
+                    worker_session_id: claim.worker_session_id,
+                    result_id: resultId,
+                    idempotency_key: `attempt:${claim.attempt_id}:terminal`,
+                    type: "attempt_terminal",
+                    occurred_at: now,
+                    payload: eventPayload({
+                        outcome: execution.result.outcome.kind,
+                        provider_event_count: execution.events.length,
+                        retry: canRetry,
+                    }),
+                },
+            ];
+            if (result) {
+                eventInputs.push({
+                    run_id: claim.run_id,
+                    step_id: claim.step_id,
+                    attempt_id: claim.attempt_id,
+                    worker_session_id: claim.worker_session_id,
+                    result_id: result.result_id,
+                    idempotency_key: `result:${result.result_id}:committed`,
+                    type: "result_committed",
+                    occurred_at: now,
+                    payload: eventPayload({ output_count: result.outputs.length }),
+                });
+            }
+            eventInputs.push({
+                run_id: claim.run_id,
+                step_id: claim.step_id,
+                attempt_id: claim.attempt_id,
+                worker_session_id: claim.worker_session_id,
+                result_id: resultId,
+                idempotency_key: `claim:${claim.claim_token}:released`,
+                type: "task_claim_released",
+                occurred_at: now,
+                payload: eventPayload({ terminal: !canRetry }),
+            }, {
+                run_id: claim.run_id,
+                step_id: claim.step_id,
+                attempt_id: claim.attempt_id,
+                worker_session_id: claim.worker_session_id,
+                result_id: resultId,
+                idempotency_key: `task:${claim.step_id}:attempt:${claim.attempt_id}:${nextState}`,
+                type: "task_state_changed",
+                occurred_at: now,
+                payload: eventPayload({ from: task.state, to: nextState }),
+            });
+            try {
+                return (await this.store.updateRun({
+                    run_id: claim.run_id,
+                    expected_record_version: snapshot.record_version,
+                    tasks: [updatedTask],
+                    attempts: [terminalAttempt],
+                    events: this.events(snapshot, eventInputs),
+                    ...(result ? { results: [result] } : {}),
+                })).snapshot;
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    syntheticExecution(claim, outcome, startedAt) {
+        const completedAt = iso(this.now());
+        return {
+            worker_session_id: claim.worker_session_id,
+            events: [],
+            result: {
+                run_id: claim.run_id,
+                step_id: claim.step_id,
+                attempt_id: claim.attempt_id,
+                worker_session_id: claim.worker_session_id,
+                provider: "controller",
+                provider_session_id: null,
+                started_at: startedAt,
+                completed_at: completedAt,
+                outcome,
+                outputs: [],
+                stdout_reference: null,
+                stderr_reference: null,
+            },
+        };
+    }
+    async reclaimDeadAttempt(snapshot, task, attempt, claim) {
+        const now = iso(this.now());
+        const cancelled = snapshot.run.status === "cancelling";
+        const outcome = cancelled
+            ? {
+                kind: "cancelled",
+                exit_code: null,
+                signal: null,
+                reason: snapshot.run.cancellation?.reason ?? "cancelled during resume",
+            }
+            : {
+                kind: "launch_error",
+                exit_code: null,
+                signal: null,
+                error: "owned worker was confirmed dead during resume",
+            };
+        const definition = this.definitionByStep.get(task.step_id);
+        const canRetry = !cancelled && retryAllowed(definition.retry_policy, attempt, outcome);
+        const nextState = cancelled ? "cancelled" : canRetry ? "ready" : "failed";
+        const terminalAttempt = {
+            ...attempt,
+            phase: "terminal",
+            completed_at: now,
+            terminal_outcome: outcome,
+        };
+        const updatedTask = {
+            ...task,
+            state: nextState,
+            updated_at: now,
+        };
+        await this.store.updateRun({
+            run_id: snapshot.run.run_id,
+            expected_record_version: snapshot.record_version,
+            tasks: [updatedTask],
+            attempts: [terminalAttempt],
+            events: this.events(snapshot, [
+                {
+                    run_id: snapshot.run.run_id,
+                    step_id: task.step_id,
+                    attempt_id: attempt.attempt_id,
+                    worker_session_id: attempt.worker_session_id,
+                    result_id: null,
+                    idempotency_key: `attempt:${attempt.attempt_id}:reclaimed-terminal`,
+                    type: "attempt_terminal",
+                    occurred_at: now,
+                    payload: eventPayload({
+                        outcome: outcome.kind,
+                        retry: canRetry,
+                        reconciliation: "dead",
+                    }),
+                },
+                {
+                    run_id: snapshot.run.run_id,
+                    step_id: task.step_id,
+                    attempt_id: attempt.attempt_id,
+                    worker_session_id: attempt.worker_session_id,
+                    result_id: null,
+                    idempotency_key: `claim:${claim.claim_token}:reclaimed`,
+                    type: "task_claim_released",
+                    occurred_at: now,
+                    payload: eventPayload({ expired: true, reconciled: "dead" }),
+                },
+                {
+                    run_id: snapshot.run.run_id,
+                    step_id: task.step_id,
+                    attempt_id: attempt.attempt_id,
+                    worker_session_id: attempt.worker_session_id,
+                    result_id: null,
+                    idempotency_key: `task:${task.step_id}:attempt:${attempt.attempt_id}:${nextState}`,
+                    type: "task_state_changed",
+                    occurred_at: now,
+                    payload: eventPayload({ from: task.state, to: nextState }),
+                },
+            ]),
+        });
+    }
+    /**
+     * Resume never infers death from lease expiry alone. Reclaim is allowed only
+     * after the injected reconciler explicitly confirms the owned worker is dead.
+     */
+    async resume(runId) {
+        while (true) {
+            let snapshot = await this.store.requireRun(runId);
+            if (snapshot.run.terminal !== null)
+                return snapshot;
+            let changed = false;
+            for (const task of snapshot.tasks) {
+                if (task.state !== "claimed" && task.state !== "running")
+                    continue;
+                const attempt = attemptById(snapshot, task.current_attempt_id);
+                if (!attempt || attempt.phase === "terminal")
+                    continue;
+                const claim = claimByAttempt(snapshot, attempt.attempt_id);
+                if (!claim) {
+                    throw new RunControllerError("claim_lost", `active attempt ${attempt.attempt_id} has no persisted claim`);
+                }
+                if (Date.parse(claim.lease.expires_at) > this.now().getTime()) {
+                    continue;
+                }
+                const reconciliation = await this.reconcileAttempt(attempt, claim);
+                if (reconciliation !== "dead")
+                    continue;
+                try {
+                    await this.reclaimDeadAttempt(snapshot, task, attempt, claim);
+                    changed = true;
+                    break;
+                }
+                catch (error) {
+                    if (isVersionConflict(error)) {
+                        changed = true;
+                        break;
+                    }
+                    throw error;
+                }
+            }
+            if (!changed) {
+                snapshot = await this.refreshDependencyReadiness(runId);
+                return this.finalizeRunIfTerminal(snapshot.run.run_id);
+            }
+        }
+    }
+    async requestCancellation(runId, reason = null) {
+        while (true) {
+            const snapshot = await this.store.requireRun(runId);
+            if (snapshot.run.terminal !== null)
+                return snapshot;
+            const now = iso(this.now());
+            const cancellation = snapshot.run.cancellation ?? {
+                requested_at: now,
+                reason,
+            };
+            const tasks = snapshot.tasks
+                .filter((task) => (task.state === "dependency_blocked" || task.state === "ready"))
+                .map((task) => ({
+                ...task,
+                state: "cancelled",
+                updated_at: now,
+            }));
+            const eventInputs = [];
+            if (snapshot.run.status !== "cancelling") {
+                eventInputs.push({
+                    run_id: runId,
+                    step_id: null,
+                    attempt_id: null,
+                    worker_session_id: null,
+                    result_id: null,
+                    idempotency_key: `run:${runId}:cancel-requested`,
+                    type: "run_cancel_requested",
+                    occurred_at: cancellation.requested_at,
+                    payload: eventPayload({ reason }),
+                });
+            }
+            for (const task of tasks) {
+                const prior = taskByStep(snapshot, task.step_id);
+                eventInputs.push({
+                    run_id: runId,
+                    step_id: task.step_id,
+                    attempt_id: null,
+                    worker_session_id: null,
+                    result_id: null,
+                    idempotency_key: `task:${task.step_id}:cancelled`,
+                    type: "task_state_changed",
+                    occurred_at: now,
+                    payload: eventPayload({ from: prior.state, to: "cancelled" }),
+                });
+            }
+            const run = {
+                ...snapshot.run,
+                status: "cancelling",
+                cancellation,
+                updated_at: now,
+            };
+            try {
+                const update = await this.store.updateRun({
+                    run_id: runId,
+                    expected_record_version: snapshot.record_version,
+                    run,
+                    tasks,
+                    events: this.events(snapshot, eventInputs),
+                });
+                for (const active of this.active.values()) {
+                    if (active.claim.run_id === runId) {
+                        active.controller.abort(reason ?? "run cancellation requested");
+                    }
+                }
+                return this.finalizeRunIfTerminal(runId);
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    async finalizeRunIfTerminal(runId) {
+        while (true) {
+            const snapshot = await this.store.requireRun(runId);
+            if (snapshot.run.terminal !== null)
+                return snapshot;
+            if (snapshot.tasks.some((task) => !terminalTask(task)))
+                return snapshot;
+            const now = iso(this.now());
+            const status = snapshot.run.status === "cancelling"
+                ? "cancelled"
+                : snapshot.tasks.some((task) => task.state === "failed")
+                    ? "failed"
+                    : "succeeded";
+            const reason = status === "failed"
+                ? "one or more Steps failed"
+                : status === "cancelled"
+                    ? snapshot.run.cancellation?.reason ?? "run cancelled"
+                    : null;
+            const run = {
+                ...snapshot.run,
+                status,
+                terminal: { status, reason, completed_at: now },
+                updated_at: now,
+            };
+            try {
+                return (await this.store.updateRun({
+                    run_id: runId,
+                    expected_record_version: snapshot.record_version,
+                    run,
+                    events: this.events(snapshot, [{
+                            run_id: runId,
+                            step_id: null,
+                            attempt_id: null,
+                            worker_session_id: null,
+                            result_id: null,
+                            idempotency_key: `run:${runId}:terminal:${status}`,
+                            type: "run_terminal",
+                            occurred_at: now,
+                            payload: eventPayload({ status, reason }),
+                        }]),
+                })).snapshot;
+            }
+            catch (error) {
+                if (isVersionConflict(error))
+                    continue;
+                throw error;
+            }
+        }
+    }
+    launchClaimed(claimed) {
+        const controller = new AbortController();
+        const completion = (async () => {
+            let heartbeat;
+            let latestClaim = claimed.claim;
+            try {
+                await this.markRunning(latestClaim);
+                heartbeat = setInterval(() => {
+                    void this.renewClaim(latestClaim)
+                        .then((renewed) => {
+                        latestClaim = renewed;
+                    })
+                        .catch((error) => {
+                        controller.abort(error instanceof Error
+                            ? error.message
+                            : "task lease renewal failed");
+                    });
+                }, this.leaseHeartbeatMs);
+                heartbeat.unref();
+                const packet = await this.materializePacket({
+                    ...claimed,
+                    claim: latestClaim,
+                });
+                const provider = this.providerForStep(claimed.definition);
+                const execution = await this.executeAttempt(packet, provider, {
+                    ...this.workerOptions,
+                    cancellationSignal: controller.signal,
+                    cancellationReason: controller.signal.aborted
+                        ? String(controller.signal.reason ?? "controller cancellation")
+                        : null,
+                    now: this.now,
+                });
+                await this.finalizeAttempt(latestClaim, execution);
+            }
+            catch (error) {
+                const snapshot = await this.store.requireRun(latestClaim.run_id);
+                const attempt = attemptById(snapshot, latestClaim.attempt_id);
+                if (attempt && attempt.phase !== "terminal") {
+                    const outcome = snapshot.run.status === "cancelling"
+                        ? {
+                            kind: "cancelled",
+                            exit_code: null,
+                            signal: null,
+                            reason: snapshot.run.cancellation?.reason
+                                ?? "run cancellation requested",
+                        }
+                        : {
+                            kind: "launch_error",
+                            exit_code: null,
+                            signal: null,
+                            error: error instanceof Error
+                                ? error.message
+                                : String(error),
+                        };
+                    const synthetic = this.syntheticExecution(latestClaim, outcome, attempt.started_at ?? iso(this.now()));
+                    await this.finalizeAttempt(latestClaim, synthetic);
+                }
+            }
+            finally {
+                if (heartbeat)
+                    clearInterval(heartbeat);
+                this.active.delete(claimed.attempt.attempt_id);
+            }
+        })();
+        this.active.set(claimed.attempt.attempt_id, {
+            claim: claimed.claim,
+            controller,
+            completion,
+        });
+        return completion;
+    }
+    /**
+     * Runs scheduling until durable terminal state. Every successful claim creates
+     * a fresh attempt and worker session; this Controller never gives a worker a
+     * second Step.
+     */
+    async runUntilTerminal(runId) {
+        while (true) {
+            let snapshot = await this.resume(runId);
+            snapshot = await this.refreshDependencyReadiness(runId);
+            snapshot = await this.finalizeRunIfTerminal(runId);
+            if (snapshot.run.terminal !== null)
+                return snapshot;
+            let launched = false;
+            while (this.active.size < this.maxConcurrency) {
+                const claimed = await this.claimNextReady(runId);
+                if (!claimed)
+                    break;
+                launched = true;
+                void this.launchClaimed(claimed);
+            }
+            if (this.active.size > 0) {
+                await Promise.race([...this.active.values()].map((entry) => entry.completion));
+                continue;
+            }
+            snapshot = await this.store.requireRun(runId);
+            const externallyActive = snapshot.tasks.some((task) => task.state === "claimed" || task.state === "running");
+            const retryPending = snapshot.tasks.some((task) => {
+                if (task.state !== "ready")
+                    return false;
+                const definition = this.definitionByStep.get(task.step_id);
+                return definition
+                    ? this.readyAt(definition, task, snapshot) > this.now().getTime()
+                    : false;
+            });
+            if (externallyActive || retryPending || launched) {
+                await sleep(this.schedulerPollMs);
+                continue;
+            }
+            throw new RunControllerError("scheduler_stalled", `run ${runId} has no runnable or active Step`);
+        }
+    }
+}