@wardby/cli 0.6.0 → 0.7.0

package/dist/providers/executor/dbos-status.d.ts

@@ -2,6 +2,7 @@
  * Pure decision table for reconciling a stale `Run` that a DBOS workflow
  * owns. Kept free of DBOS imports so it is unit-testable without Postgres.
  */
+import { STEPS_CHANGED_REASON } from "../engine/types.js";
 export interface WorkflowStatusLike {
     status: string;
     executorId?: string;
@@ -48,3 +49,33 @@ export type RecoveryDecision = {
     reason: string;
 };
 export declare function decideRecovery(status: WorkflowStatusLike | null, runIsTerminal: boolean, selfExecutorId: string): RecoveryDecision;
+/** One recorded step of a workflow, as `DBOS.listWorkflowSteps` reports it. */
+export interface TakeoverStep {
+    functionID: number;
+    name: string;
+    output: unknown;
+}
+/** A still-drivable child of the lead being taken over. */
+export interface TakeoverChild {
+    status: string;
+    kind: "coding" | "native";
+    nativeExecutionMode: string | null;
+}
+export type TakeoverDecision = {
+    action: "fork";
+    startStep: number;
+} | {
+    action: "lost";
+    reason: string;
+};
+/** Why a forked run ends lost when its replay meets a step the new code no longer takes (defined with StepReplayMismatchError). */
+export { STEPS_CHANGED_REASON };
+/**
+ * Whether a workflow recorded under another application version can be forked under this one
+ * (design §1): only when it is waiting inside a `delegate_to_*` call of its last recorded turn and
+ * every child it may be waiting on is durable on its own (a coding run, or a sandbox-mode native
+ * run). The fork starts at the first function id with no record, so every recorded step (the
+ * turn's model reply above all) replays from its record and the delegate step re-attaches to its
+ * child. Anything else is `lost` with the mismatch reason, as before.
+ */
+export declare function decideTakeover(steps: TakeoverStep[], children: TakeoverChild[], mismatchReason: string): TakeoverDecision;

package/dist/providers/executor/dbos-status.js

@@ -2,6 +2,8 @@
  * Pure decision table for reconciling a stale `Run` that a DBOS workflow
  * owns. Kept free of DBOS imports so it is unit-testable without Postgres.
  */
+import { DELEGATE_TOOL_PREFIX } from "../../core/tool-names.js";
+import { STEPS_CHANGED_REASON } from "../engine/types.js";
 /**
  * How many times the reconciler will try to adopt (resume) one orphaned
  * workflow before declaring the run lost.
@@ -62,3 +64,65 @@ export function decideRecovery(status, runIsTerminal, selfExecutorId) {
     }
     return { action: "lost", reason: `Durable workflow is in unrecognised status "${status.status}".` };
 }
+/** Why a forked run ends lost when its replay meets a step the new code no longer takes (defined with StepReplayMismatchError). */
+export { STEPS_CHANGED_REASON };
+const LLM_STEP = /^turn:(\d+):llm$/;
+/**
+ * The tool-call names of a recorded `turn:N:llm` step. Its output is the engine's TurnResult
+ * (engine-native.ts) as DBOS deserialized it: `{ usage, text, toolCalls: [{ id, name, argsJson }],
+ * budgetExceededMidStream, error? }`. Anything else yields undefined (the shape changed: lost).
+ */
+function recordedToolCallNames(output) {
+    if (typeof output !== "object" || output === null || !("toolCalls" in output))
+        return undefined;
+    const calls = output.toolCalls;
+    if (!Array.isArray(calls))
+        return undefined;
+    const names = [];
+    for (const call of calls) {
+        if (typeof call !== "object" || call === null || !("name" in call) || typeof call.name !== "string") {
+            return undefined;
+        }
+        names.push(call.name);
+    }
+    return names;
+}
+/**
+ * Whether a workflow recorded under another application version can be forked under this one
+ * (design §1): only when it is waiting inside a `delegate_to_*` call of its last recorded turn and
+ * every child it may be waiting on is durable on its own (a coding run, or a sandbox-mode native
+ * run). The fork starts at the first function id with no record, so every recorded step (the
+ * turn's model reply above all) replays from its record and the delegate step re-attaches to its
+ * child. Anything else is `lost` with the mismatch reason, as before.
+ */
+export function decideTakeover(steps, children, mismatchReason) {
+    const lost = { action: "lost", reason: mismatchReason };
+    const sorted = [...steps].sort((a, b) => a.functionID - b.functionID);
+    const llmStep = sorted.filter((s) => LLM_STEP.test(s.name)).at(-1);
+    if (!llmStep)
+        return lost;
+    const turn = LLM_STEP.exec(llmStep.name)[1];
+    const names = recordedToolCallNames(llmStep.output);
+    if (!names)
+        return lost;
+    const recorded = new Set(sorted.map((s) => s.name));
+    const firstPending = names.findIndex((_, i) => !recorded.has(`turn:${turn}:tool:${i}`));
+    // Not waiting in a tool call at all, or waiting in one that is not a delegation (re-running it
+    // under new code would repeat its effect).
+    if (firstPending < 0 || !names[firstPending].startsWith(DELEGATE_TOOL_PREFIX))
+        return lost;
+    const durable = (child) => child.kind === "coding" || child.nativeExecutionMode === "sandbox";
+    if (!children.every(durable))
+        return lost;
+    let startStep = 0;
+    for (const step of sorted) {
+        if (step.functionID > startStep)
+            break;
+        if (step.functionID === startStep)
+            startStep += 1;
+    }
+    // A gap at or before the recorded reply would re-run the model turn: never fork into that.
+    if (startStep <= llmStep.functionID)
+        return lost;
+    return { action: "fork", startStep };
+}

package/dist/providers/executor/dbos.d.ts

@@ -15,7 +15,10 @@
  * A stalled-but-alive owner can overlap an adopted attempt of the same run:
  * whichever records a step second loses ownership, stops at that step
  * without writing the run, and hands the SDK's conflict back so DBOS parks
- * it as a duplicate execution (see dbosStep and runInsideWorkflow).
+ * it as a duplicate execution (see dbosStep and runInsideWorkflow). After a
+ * takeover the fork has its own workflow id, so the two never share a step:
+ * the original stops the same way when it meets the takeover's cancel
+ * (dbosStepFor), and its start() handle writes nothing.
  */
 import pg from "pg";
 import type { PrismaClient } from "#prisma";
@@ -44,6 +47,20 @@ export declare function dbosLaunchConfig(config: DbosConfig, executorId: string,
     logLevel: "warn";
     systemDatabasePool?: pg.Pool;
 };
+/**
+ * The id a takeover gives the fork of workflow `from` under application `version`. Deterministic, so
+ * a takeover that forked but never recorded the fork finishes it rather than forking again; and
+ * descendants share their ancestor's id as a prefix, which isForkOf relies on.
+ */
+export declare function forkWorkflowId(from: string, version: string): string;
+/** Whether workflow `id` was forked from `from`, directly or through further forks (forkWorkflowId). */
+export declare function isForkOf(id: string, from: string): boolean;
+/**
+ * The DBOS workflow id that drives a run: `Run.executionWorkflowId` once the
+ * run's workflow was forked under a new application version, otherwise the
+ * run id (a run's first workflow is started with its run id).
+ */
+export declare function workflowIdOf(db: Pick<PrismaClient, "run">, runId: string): Promise<string>;
 /**
  * Bind the engine's step boundary to a DBOS checkpointed step. Never
  * retried: a retry would re-spend.
@@ -59,8 +76,34 @@ export declare function dbosLaunchConfig(config: DbosConfig, executorId: string,
  * discarded step's usage, if any, for the log) so executeRun's backstop stops
  * this attempt without writing the run; runInsideWorkflow turns it back into
  * the SDK error.
+ *
+ * DBOS signals that a replay reached a step whose recorded name differs (the
+ * run was recorded by a release whose steps differ, e.g. after a takeover
+ * fork) by throwing `DBOSUnexpectedStepError`. Translated into a
+ * `StepReplayMismatchError`, which executeRun's backstop settles as `lost`
+ * (runInsideWorkflow does it for a mismatch at the load step, which is outside
+ * that backstop).
  */
 export declare const dbosStep: StepRunner;
+/**
+ * dbosStep for one run's workflow body. A cancellation of a workflow the run has moved off (its
+ * `executionWorkflowId` names a fork of it) is not a stop: recover() took the run over (forked it) and cancelled this,
+ * the forked-from workflow, while its owner was still alive. That attempt stops like one that lost
+ * a step's checkpoint (RunOwnershipLostError: no terminal write), so the fork, not a reasonless
+ * "cancelled" from the original, settles the run. takeOver records the fork's id before it
+ * cancels, so the original always reads the new id by the time it sees the cancel.
+ */
+export declare function dbosStepFor(db: Pick<PrismaClient, "run">, runId: string): StepRunner;
+/**
+ * dbosStepFor behind the same per-step row read as inline runs (stopAwareStep), failing open. DBOS
+ * stops a workflow it was told to cancel, but stop() can cancel only the workflow the run names, and
+ * a takeover's fork is named only once recorded: a fork already running before that (or whose
+ * record failed) would otherwise keep paying for turns under a run stop() already made terminal.
+ * The same read fences out a taken-over original as soon as the run names its fork, before the
+ * takeover's cancel lands, and even when that cancel failed: it stops as an ownership loss, without
+ * a terminal write or the settle side effects (host checks, status comments) the fork owns.
+ */
+export declare function stopAwareDbosStep(db: Pick<PrismaClient, "run">, runId: string): StepRunner;
 export declare class DbosExecutor implements Executor {
     private readonly providers;
     private readonly config;
@@ -90,6 +133,12 @@ export declare class DbosExecutor implements Executor {
      * the same thing launch() does for this executor's own workflows.
      */
     recover(handle: PersistedExecutionHandle): Promise<ExecutionRecoveryResult>;
+    /**
+     * A workflow recorded under another application version: fork it under this one when it is only
+     * waiting on durable children (decideTakeover), otherwise answer lost with the mismatch. The
+     * fork counts as an adoption attempt, so repeated failed takeovers of one run end lost too.
+     */
+    private takeOver;
     /** Workflow body. Public only so the module-level registration can reach it. */
     runInsideWorkflow(runId: string): Promise<void>;
 }

package/dist/providers/executor/dbos.js

@@ -15,18 +15,22 @@
  * A stalled-but-alive owner can overlap an adopted attempt of the same run:
  * whichever records a step second loses ownership, stops at that step
  * without writing the run, and hands the SDK's conflict back so DBOS parks
- * it as a duplicate execution (see dbosStep and runInsideWorkflow).
+ * it as a duplicate execution (see dbosStep and runInsideWorkflow). After a
+ * takeover the fork has its own workflow id, so the two never share a step:
+ * the original stops the same way when it meets the takeover's cancel
+ * (dbosStepFor), and its start() handle writes nothing.
  */
 import { DBOS, Error as DbosErrors } from "@dbos-inc/dbos-sdk";
 import pg from "pg";
-import { RunOwnershipLostError } from "../engine/types.js";
-import { executeRun, RunCancelledError } from "../../core/runner.js";
+import { RunOwnershipLostError, StepReplayMismatchError } from "../engine/types.js";
+import { executeRun, failNativeRun, loseNativeRun, nativeRunIntegrations, RunCancelledError, stopAwareStep, } from "../../core/runner.js";
 import { markRunFailedFromExecutorError } from "../../core/dispatch.js";
 import { prisma as defaultDb } from "../../core/db.js";
 import { HEARTBEAT_INTERVAL_MS } from "../../core/timing.js";
+import { DRIVABLE } from "../../core/run-status.js";
 import { logger } from "../../core/logger.js";
 import { databaseAuthFromEnv, pgConnectionConfig } from "../../core/pg-auth.js";
-import { ADOPTION_EXHAUSTED_REASON, decideRecovery, shouldGiveUpAdoption, versionMismatchReason, } from "./dbos-status.js";
+import { ADOPTION_EXHAUSTED_REASON, STEPS_CHANGED_REASON, decideRecovery, decideTakeover, shouldGiveUpAdoption, versionMismatchReason, } from "./dbos-status.js";
 export const DBOS_BACKEND = "dbos";
 const dbosLog = logger.child({ module: "dbos-executor" });
 /** DBOS's own system pool size (DEFAULT_POOL_SIZE in the SDK's system_database.js); kept for ours. */
@@ -87,6 +91,27 @@ function setActiveExecutor(executor) {
  * `runInsideWorkflow`'s finally.
  */
 const cancellationReasons = new Map();
+/**
+ * The id a takeover gives the fork of workflow `from` under application `version`. Deterministic, so
+ * a takeover that forked but never recorded the fork finishes it rather than forking again; and
+ * descendants share their ancestor's id as a prefix, which isForkOf relies on.
+ */
+export function forkWorkflowId(from, version) {
+    return `${from}@${version}`;
+}
+/** Whether workflow `id` was forked from `from`, directly or through further forks (forkWorkflowId). */
+export function isForkOf(id, from) {
+    return id.startsWith(`${from}@`);
+}
+/**
+ * The DBOS workflow id that drives a run: `Run.executionWorkflowId` once the
+ * run's workflow was forked under a new application version, otherwise the
+ * run id (a run's first workflow is started with its run id).
+ */
+export async function workflowIdOf(db, runId) {
+    const row = await db.run.findUnique({ where: { id: runId }, select: { executionWorkflowId: true } });
+    return row?.executionWorkflowId ?? runId;
+}
 /**
  * Bind the engine's step boundary to a DBOS checkpointed step. Never
  * retried: a retry would re-spend.
@@ -102,8 +127,47 @@ const cancellationReasons = new Map();
  * discarded step's usage, if any, for the log) so executeRun's backstop stops
  * this attempt without writing the run; runInsideWorkflow turns it back into
  * the SDK error.
+ *
+ * DBOS signals that a replay reached a step whose recorded name differs (the
+ * run was recorded by a release whose steps differ, e.g. after a takeover
+ * fork) by throwing `DBOSUnexpectedStepError`. Translated into a
+ * `StepReplayMismatchError`, which executeRun's backstop settles as `lost`
+ * (runInsideWorkflow does it for a mismatch at the load step, which is outside
+ * that backstop).
+ */
+export const dbosStep = (name, fn) => runDbosStep(name, fn);
+/**
+ * dbosStep for one run's workflow body. A cancellation of a workflow the run has moved off (its
+ * `executionWorkflowId` names a fork of it) is not a stop: recover() took the run over (forked it) and cancelled this,
+ * the forked-from workflow, while its owner was still alive. That attempt stops like one that lost
+ * a step's checkpoint (RunOwnershipLostError: no terminal write), so the fork, not a reasonless
+ * "cancelled" from the original, settles the run. takeOver records the fork's id before it
+ * cancels, so the original always reads the new id by the time it sees the cancel.
+ */
+export function dbosStepFor(db, runId) {
+    const supersedes = async (cancelledWorkflowId) => {
+        // A failed read falls back to an ordinary cancellation, as before this check existed.
+        const current = await workflowIdOf(db, runId).catch(() => cancelledWorkflowId);
+        return isForkOf(current, cancelledWorkflowId);
+    };
+    return (name, fn) => runDbosStep(name, fn, supersedes);
+}
+/**
+ * dbosStepFor behind the same per-step row read as inline runs (stopAwareStep), failing open. DBOS
+ * stops a workflow it was told to cancel, but stop() can cancel only the workflow the run names, and
+ * a takeover's fork is named only once recorded: a fork already running before that (or whose
+ * record failed) would otherwise keep paying for turns under a run stop() already made terminal.
+ * The same read fences out a taken-over original as soon as the run names its fork, before the
+ * takeover's cancel lands, and even when that cancel failed: it stops as an ownership loss, without
+ * a terminal write or the settle side effects (host checks, status comments) the fork owns.
  */
-export const dbosStep = async (name, fn) => {
+export function stopAwareDbosStep(db, runId) {
+    return stopAwareStep(db, runId, dbosStepFor(db, runId), {
+        failOpen: true,
+        ownership: { workflowId: () => DBOS.workflowID, forkedFrom: isForkOf },
+    });
+}
+async function runDbosStep(name, fn, supersededBy) {
     let ran;
     try {
         return await DBOS.runStep(async () => {
@@ -114,6 +178,8 @@ export const dbosStep = async (name, fn) => {
     }
     catch (err) {
         if (err instanceof DbosErrors.DBOSWorkflowCancelledError) {
+            if (await supersededBy?.(err.workflowID))
+                throw new RunOwnershipLostError(name, { cause: err });
             const reason = cancellationReasons.get(err.workflowID);
             throw new RunCancelledError(`Run cancelled: ${reason ?? "no reason given"}`);
         }
@@ -121,9 +187,11 @@ export const dbosStep = async (name, fn) => {
             const discardedUsage = usageOf(ran?.value);
             throw new RunOwnershipLostError(name, { cause: err, ...(discardedUsage ? { discardedUsage } : {}) });
         }
+        if (err instanceof DbosErrors.DBOSUnexpectedStepError)
+            throw new StepReplayMismatchError(name, { cause: err });
         throw err;
     }
-};
+}
 /**
  * The numeric `usage` fields of a step result, if it has any (an LLM turn's
  * token counts and cost). Tool steps return a string and yield nothing.
@@ -141,6 +209,19 @@ function usageOf(value) {
  * Registered once at module load — DBOS requires registration before
  * launch(). The body must be deterministic between steps; everything
  * non-deterministic in executeRun/engine is inside a runStep.
+ *
+ * Replay constraints. A run's workflow is replayed both by a same-version
+ * resume and, when recover() takes a waiting lead over after a deploy, by a
+ * fork that runs this body under the *new* release against steps the *old*
+ * release recorded (see decideTakeover). So:
+ * - Code between steps must stay deterministic and safe to repeat: it runs
+ *   again on every replay, now possibly under newer code.
+ * - Step names and their order must only change in ways a replay can still
+ *   follow; a replay that meets a different recorded name ends the run lost
+ *   with STEPS_CHANGED_REASON (executeRun's backstop, or runInsideWorkflow).
+ * - New code must accept step outputs recorded by the previous release (the
+ *   `turn:N:llm` TurnResult, tool results). A release that changes a recorded
+ *   shape keeps reading the old one, or a takeover fails at that step (lost).
  */
 const runWorkflow = DBOS.registerWorkflow(async (runId) => {
     const self = activeExecutor;
@@ -218,12 +299,16 @@ export class DbosExecutor {
         if (!this.launched)
             return;
         this.launched = false;
-        if (activeExecutor === this)
-            setActiveExecutor(undefined);
         try {
             await DBOS.shutdown({ workflowCompletionTimeoutMS: 5_000 });
         }
         finally {
+            // Only now: shutdown first lets the queue runner finish the dispatch it is in, so a workflow
+            // the queue claimed just before (a takeover's fork, an adoption) still starts. Without its
+            // executor its body would throw, and DBOS could record that as the workflow's outcome: the
+            // next process would then mark a run failed that was only waiting to be taken over.
+            if (activeExecutor === this)
+                setActiveExecutor(undefined);
             await endSystemPool();
         }
     }
@@ -263,6 +348,15 @@ export class DbosExecutor {
             // the same failure — dispatchRun's own `.catch` around `start()` calls
             // it too — which is harmless precisely because it is conditional and
             // idempotent: the second call matches zero rows.
+            //
+            // A workflow that recover() took over (forked, then cancelled) no longer drives the run: the
+            // fork does, and settles it. Its rejection is the takeover's cancel, not the run's failure.
+            if ((await workflowIdOf(this.db, runId).catch(() => runId)) !== runId) {
+                // The error's name only: this path adds no message text to any output.
+                const error = err instanceof Error ? err.name : typeof err;
+                dbosLog.info({ runId, error }, "the run's original workflow ended after a takeover; its fork drives the run");
+                return;
+            }
             await markRunFailedFromExecutorError(this.db, runId, err, this.selfDefects);
             throw err;
         }
@@ -274,11 +368,43 @@ export class DbosExecutor {
         if (!this.launched)
             await this.launch();
         dbosLog.info({ runId, reason }, "cancelling durable run");
+        // The terminal write comes first and is conditional (the first writer wins), so the run is
+        // cancelled with the caller's reason even when its workflow runs in another process (where the
+        // reason map below is empty) or has no workflow at all yet (cancelWorkflow then matches
+        // nothing). The workflow's next step then throws, and its own terminal write matches no row.
+        // The cancel is still attempted when that settle throws (a host or issue update failing after
+        // the write) or the database is unreachable: a cancelled row whose workflow kept going would
+        // keep spending. The settle's error, if any, is the one rethrown.
+        let settleFailed;
+        try {
+            const row = await this.db.run.findUnique({ where: { id: runId }, select: { status: true } });
+            if (row && (row.status === "pending" || row.status === "running")) {
+                await failNativeRun({ runId, db: this.db, providers: this.providers, ...nativeRunIntegrations(this.providers, this.db) }, new RunCancelledError(reason ?? "The run was cancelled."));
+            }
+        }
+        catch (err) {
+            settleFailed = { err };
+        }
         // Recorded before the cancel so the workflow's next step boundary can
         // find it: DBOS only reports "cancelled", never why.
-        if (reason)
+        // Keyed by both ids: dbosStep looks it up by the cancelled workflow's id, which is the forked
+        // id for a forked run. A failed read targets the run id, the workflow of any run never forked.
+        const workflowId = await workflowIdOf(this.db, runId).catch(() => runId);
+        if (reason) {
             cancellationReasons.set(runId, reason);
-        await DBOS.cancelWorkflow(runId);
+            cancellationReasons.set(workflowId, reason);
+        }
+        try {
+            await DBOS.cancelWorkflow(workflowId);
+        }
+        catch (err) {
+            if (!settleFailed)
+                throw err;
+            // The error's name only: this path adds no message text to any output.
+            dbosLog.warn({ runId, error: err instanceof Error ? err.name : typeof err }, "could not cancel the workflow");
+        }
+        if (settleFailed)
+            throw settleFailed.err;
     }
     /**
      * Answer the reconciler for a stale run. Never relaunches: `resume`
@@ -307,13 +433,12 @@ export class DbosExecutor {
                 // DBOS gates recovery and dequeue on application_version. A workflow
                 // recorded under a different version is never dequeued here, so
                 // resuming it would be an unbounded loop that holds the run
-                // `running` forever: give up at once and let the reconciler reap it.
+                // `running` forever. Instead it is forked under this version when it
+                // is only waiting on durable children, and otherwise given up at once
+                // so the reconciler reaps it (takeOver).
                 const mismatch = versionMismatchReason(status?.applicationVersion, DBOS.applicationVersion);
-                if (mismatch) {
-                    this.resumeAttempts.delete(handle.runId);
-                    dbosLog.warn({ runId: handle.runId, workflowVersion: status?.applicationVersion }, mismatch);
-                    return { state: "lost", reason: mismatch };
-                }
+                if (mismatch)
+                    return this.takeOver(handle, mismatch, status?.applicationVersion);
                 // Belt and braces for every other reason a resume can fail to take
                 // (an id we can't dequeue for, a queue that never drains): bound the
                 // number of adoptions of one run so the reconciler stops re-driving
@@ -347,6 +472,56 @@ export class DbosExecutor {
                 return { state: "lost", reason: decision.reason };
         }
     }
+    /**
+     * A workflow recorded under another application version: fork it under this one when it is only
+     * waiting on durable children (decideTakeover), otherwise answer lost with the mismatch. The
+     * fork counts as an adoption attempt, so repeated failed takeovers of one run end lost too.
+     */
+    async takeOver(handle, mismatch, workflowVersion) {
+        const steps = (await DBOS.listWorkflowSteps(handle.id)) ?? [];
+        const children = await this.db.run.findMany({
+            where: { parentRunId: handle.runId, status: { in: [...DRIVABLE] } },
+            select: { status: true, nativeExecutionMode: true, agent: { select: { kind: true } } },
+        });
+        const decision = decideTakeover(steps.map((s) => ({ functionID: s.functionID, name: s.name, output: s.output })), children.map((c) => ({ status: c.status, kind: c.agent.kind, nativeExecutionMode: c.nativeExecutionMode })), mismatch);
+        if (decision.action === "lost") {
+            this.resumeAttempts.delete(handle.runId);
+            dbosLog.warn({ runId: handle.runId, workflowVersion }, mismatch);
+            return { state: "lost", reason: decision.reason };
+        }
+        const attempts = this.resumeAttempts.get(handle.runId) ?? 0;
+        if (shouldGiveUpAdoption(attempts)) {
+            this.resumeAttempts.delete(handle.runId);
+            dbosLog.warn({ runId: handle.runId, attempts }, "giving up on taking over durable run");
+            return { state: "lost", reason: ADOPTION_EXHAUSTED_REASON };
+        }
+        this.resumeAttempts.set(handle.runId, attempts + 1);
+        // Deterministic, so a takeover that forked but never recorded the fork (the run write below
+        // failed, or the process died) is finished by the next pass rather than forked a second time:
+        // two workflows driving one run would each spend on every turn. DBOS inserts a fork's row with
+        // a plain INSERT (bulkForkWorkflows in the SDK's system_database.js), so a repeated id would
+        // fail on the key: look first, and treat a lost race the same way.
+        const forkedId = forkWorkflowId(handle.id, DBOS.applicationVersion);
+        if (!(await DBOS.getWorkflowStatus(forkedId))) {
+            try {
+                await DBOS.forkWorkflow(handle.id, decision.startStep, {
+                    newWorkflowID: forkedId,
+                    applicationVersion: DBOS.applicationVersion,
+                });
+            }
+            catch (err) {
+                if (!(await DBOS.getWorkflowStatus(forkedId)))
+                    throw err;
+            }
+        }
+        const forked = DBOS.retrieveWorkflow(forkedId);
+        await this.db.run.update({ where: { id: handle.runId }, data: { executionWorkflowId: forkedId } });
+        // So nothing can dequeue the original again (it stays recorded as forked from).
+        await DBOS.cancelWorkflow(handle.id).catch((err) => dbosLog.warn({ err, runId: handle.runId }, "could not cancel the forked-from workflow"));
+        dbosLog.info({ runId: handle.runId, from: handle.id, to: forkedId, startStep: decision.startStep, workflowVersion }, "run taken over under this application version");
+        void forked.getResult().catch((err) => dbosLog.error({ err, runId: handle.runId }, "taken-over run failed"));
+        return { state: "active" };
+    }
     /** Workflow body. Public only so the module-level registration can reach it. */
     async runInsideWorkflow(runId) {
         const beat = () => this.db.run.update({ where: { id: runId }, data: { heartbeatAt: new Date() } }).catch(() => {
@@ -355,23 +530,35 @@ export class DbosExecutor {
         await beat();
         const timer = setInterval(() => void beat(), this.heartbeatIntervalMs);
         try {
-            await executeRun(runId, this.providers, this.db, undefined, dbosStep);
+            await executeRun(runId, this.providers, this.db, undefined, stopAwareDbosStep(this.db, runId));
         }
         catch (err) {
             // An ownership loss leaves the workflow body as the SDK's own conflict error, so DBOS takes
             // its duplicate-execution path: it records nothing and parks this attempt until the winner
             // records the outcome. Any other error escaping here would be recorded as the workflow's
-            // outcome while the winner still runs. The runner already logged the loss.
+            // outcome while the winner still runs. The runner already logged the loss. A forked-from
+            // workflow's own cancellation (dbosStepFor) goes back as itself: DBOS parks that too.
             if (err instanceof RunOwnershipLostError) {
-                throw err.cause instanceof DbosErrors.DBOSWorkflowConflictError
+                throw err.cause instanceof DbosErrors.DBOSWorkflowConflictError ||
+                    err.cause instanceof DbosErrors.DBOSWorkflowCancelledError
                     ? err.cause
-                    : new DbosErrors.DBOSWorkflowConflictError(runId);
+                    : new DbosErrors.DBOSWorkflowConflictError(DBOS.workflowID ?? runId);
+            }
+            // A replay met a step the recorded run does not have at the load step, ahead of executeRun's
+            // backstop (which settles a mismatch in any later step itself).
+            if (err instanceof StepReplayMismatchError) {
+                dbosLog.warn({ err, runId }, "replay no longer matches the recorded run");
+                // The same settle as executeRun's backstop: lost (first writer wins), its children stopped.
+                await loseNativeRun({ runId, db: this.db, providers: this.providers, ...nativeRunIntegrations(this.providers, this.db) }, STEPS_CHANGED_REASON);
+                return;
             }
             throw err;
         }
         finally {
             clearInterval(timer);
             cancellationReasons.delete(runId);
+            if (DBOS.workflowID)
+                cancellationReasons.delete(DBOS.workflowID);
         }
     }
 }

package/dist/providers/executor/deferred.d.ts

@@ -6,43 +6,31 @@
  * stop it needs only requested (Run.stopRequestedAt). The scheduler leader
  * carries both out through the server's own executor (`drainDeferredRuns`).
  *
- * Coding dispatch still needs the worker image and the repository's service
- * declaration and knowledge files, resolved at dispatch: those come from a
- * resolve-only view of the configured coding executor, whose start/stop are
- * never called here.
+ * Coding dispatch is recorded unresolved (`defersCodingResolution`): the worker image,
+ * service declaration and knowledge files are resolved by the process that starts the run
+ * (the drain's `beforeStart` hook), not here.
  */
 import type { PrismaClient } from "#prisma";
-import type { CodingImageSelector, Executor } from "./types.js";
-import type { CodingProvider } from "../../coding/provider.js";
+import type { Executor } from "./types.js";
 export type DeferredExecutorDb = Pick<PrismaClient, "run">;
 export declare class DeferredExecutor implements Executor {
     private readonly db;
-    /** Coding resolution only (image, service declaration, repository files); never started or stopped. */
-    private readonly resolver?;
-    constructor(db: DeferredExecutorDb, 
-    /** Coding resolution only (image, service declaration, repository files); never started or stopped. */
-    resolver?: Executor | undefined);
+    readonly defersCodingResolution = true;
+    constructor(db: DeferredExecutorDb);
     start(runId: string): Promise<void>;
     stop(runId: string, reason?: string): Promise<void>;
-    resolveCodingWorkerImage(selector: CodingImageSelector): string;
-    resolveCodingToolImage(selector: CodingImageSelector): string | null;
-    readCodingServiceDeclaration(input: {
-        repository: string;
-        baseRef: string;
-    }): Promise<string | null>;
-    readCodingRepositoryFile(input: {
-        repository: string;
-        baseRef: string;
-        path: string;
-        maxBytes: number;
-    }): Promise<string | null>;
-    supportsCodingServices(provider: CodingProvider): boolean;
 }
 export interface DrainDeferredRunsOptions {
     db: Pick<PrismaClient, "run">;
     executor: Executor;
     /** Where a start failure is recorded; defaults to marking the run failed with the error. */
     onStartFailed?: (runId: string, err: unknown) => Promise<void>;
+    /**
+     * Runs after a row is claimed and before `executor.start`. Resolves inputs the deferring process
+     * could not; false means the hook ended the run itself so it must not start. A throw is handled
+     * like a start failure.
+     */
+    beforeStart?: (runId: string) => Promise<boolean>;
     limit?: number;
 }
 /**

package/dist/providers/executor/deferred.js

@@ -6,21 +6,17 @@
  * stop it needs only requested (Run.stopRequestedAt). The scheduler leader
  * carries both out through the server's own executor (`drainDeferredRuns`).
  *
- * Coding dispatch still needs the worker image and the repository's service
- * declaration and knowledge files, resolved at dispatch: those come from a
- * resolve-only view of the configured coding executor, whose start/stop are
- * never called here.
+ * Coding dispatch is recorded unresolved (`defersCodingResolution`): the worker image,
+ * service declaration and knowledge files are resolved by the process that starts the run
+ * (the drain's `beforeStart` hook), not here.
  */
 import { logger } from "../../core/logger.js";
 const deferredLog = logger.child({ module: "deferred-executor" });
 export class DeferredExecutor {
     db;
-    resolver;
-    constructor(db, 
-    /** Coding resolution only (image, service declaration, repository files); never started or stopped. */
-    resolver) {
+    defersCodingResolution = true;
+    constructor(db) {
         this.db = db;
-        this.resolver = resolver;
     }
     async start(runId) {
         await this.db.run.updateMany({ where: { id: runId, status: "pending" }, data: { startDeferredAt: new Date() } });
@@ -43,25 +39,6 @@ export class DeferredExecutor {
             data: { stopRequestedAt: new Date(), stopRequestReason: reason ?? null },
         });
     }
-    resolveCodingWorkerImage(selector) {
-        if (!this.resolver?.resolveCodingWorkerImage)
-            throw new Error("coding_execution_not_configured");
-        return this.resolver.resolveCodingWorkerImage(selector);
-    }
-    resolveCodingToolImage(selector) {
-        if (!this.resolver?.resolveCodingToolImage)
-            throw new Error("coding_execution_not_configured");
-        return this.resolver.resolveCodingToolImage(selector);
-    }
-    async readCodingServiceDeclaration(input) {
-        return this.resolver?.readCodingServiceDeclaration?.(input) ?? null;
-    }
-    async readCodingRepositoryFile(input) {
-        return this.resolver?.readCodingRepositoryFile?.(input) ?? null;
-    }
-    supportsCodingServices(provider) {
-        return this.resolver?.supportsCodingServices?.(provider) === true;
-    }
 }
 /**
  * Starts runs a deferred executor dispatched, and stops runs it asked to stop, through the
@@ -88,7 +65,11 @@ export async function drainDeferredRuns(options) {
             continue;
         started += 1;
         // Not awaited: an in-process executor's start runs the whole run.
-        void executor.start(id).catch(async (err) => {
+        void (async () => {
+            if (options.beforeStart && !(await options.beforeStart(id)))
+                return;
+            await executor.start(id);
+        })().catch(async (err) => {
             deferredLog.error({ err, runId: id }, "deferred run failed to start");
             await (options.onStartFailed?.(id, err) ??
                 db.run.updateMany({