@wardby/cli 0.5.3 → 0.7.0

package/dist/providers/executor/dbos.js

@@ -11,16 +11,71 @@
  * The heartbeat is kept while the workflow runs here so the reconciler's
  * stale-heartbeat detection is unchanged; the reconciler consults
  * `recover()` (Task 6) before declaring a DBOS run lost.
+ *
+ * A stalled-but-alive owner can overlap an adopted attempt of the same run:
+ * whichever records a step second loses ownership, stops at that step
+ * without writing the run, and hands the SDK's conflict back so DBOS parks
+ * it as a duplicate execution (see dbosStep and runInsideWorkflow). After a
+ * takeover the fork has its own workflow id, so the two never share a step:
+ * the original stops the same way when it meets the takeover's cancel
+ * (dbosStepFor), and its start() handle writes nothing.
  */
 import { DBOS, Error as DbosErrors } from "@dbos-inc/dbos-sdk";
-import { executeRun, RunCancelledError } from "../../core/runner.js";
+import pg from "pg";
+import { RunOwnershipLostError, StepReplayMismatchError } from "../engine/types.js";
+import { executeRun, failNativeRun, loseNativeRun, nativeRunIntegrations, RunCancelledError, stopAwareStep, } from "../../core/runner.js";
 import { markRunFailedFromExecutorError } from "../../core/dispatch.js";
 import { prisma as defaultDb } from "../../core/db.js";
 import { HEARTBEAT_INTERVAL_MS } from "../../core/timing.js";
+import { DRIVABLE } from "../../core/run-status.js";
 import { logger } from "../../core/logger.js";
-import { ADOPTION_EXHAUSTED_REASON, decideRecovery, shouldGiveUpAdoption, versionMismatchReason, } from "./dbos-status.js";
+import { databaseAuthFromEnv, pgConnectionConfig } from "../../core/pg-auth.js";
+import { ADOPTION_EXHAUSTED_REASON, STEPS_CHANGED_REASON, decideRecovery, decideTakeover, shouldGiveUpAdoption, versionMismatchReason, } from "./dbos-status.js";
 export const DBOS_BACKEND = "dbos";
 const dbosLog = logger.child({ module: "dbos-executor" });
+/** DBOS's own system pool size (DEFAULT_POOL_SIZE in the SDK's system_database.js); kept for ours. */
+const DBOS_SYSTEM_POOL_SIZE = 10;
+/** DBOS's own connect timeout when the URL sets no connect_timeout (getClientConfig in the SDK's utils.js). */
+const DBOS_CONNECT_TIMEOUT_MS = 10_000;
+/**
+ * DBOS.setConfig input. Under password auth this is exactly what launch() always passed, and DBOS
+ * opens its own pool from the URL. Under rds-iam DBOS gets a pool whose password is the IAM token;
+ * DBOS then runs its migrations, queries and LISTEN through that pool and never opens a URL
+ * connection (system_database.js: a custom pool skips database creation and is what every
+ * connection borrows from). The pool copies what DBOS would have set on its own: size, connect
+ * timeout and application_name. DBOS never ends a pool it was given — see endSystemPool().
+ */
+export function dbosLaunchConfig(config, executorId, auth, pgAuth) {
+    const url = config.systemDatabaseUrl;
+    const launch = {
+        name: "wardby",
+        systemDatabaseUrl: url,
+        systemDatabaseSchemaName: config.schemaName,
+        executorID: executorId,
+        runAdminServer: false,
+        logLevel: "warn",
+    };
+    if (auth !== "rds-iam")
+        return launch;
+    const connection = pgConnectionConfig(url, auth, pgAuth); // validates the URL first
+    const connectTimeout = Number.parseInt(new URL(url).searchParams.get("connect_timeout") ?? "", 10);
+    const pool = new pg.Pool({
+        application_name: `dbos_transact_${executorId}`,
+        ...connection,
+        max: DBOS_SYSTEM_POOL_SIZE,
+        connectionTimeoutMillis: connectTimeout ? connectTimeout * 1000 : DBOS_CONNECT_TIMEOUT_MS,
+    });
+    // DBOS attaches no listener to a caller's pool; an idle connection's error would otherwise crash the process.
+    pool.on("error", (err) => dbosLog.warn({ err }, "DBOS system database pool error"));
+    return { ...launch, systemDatabasePool: pool };
+}
+/** The pool launch() handed DBOS, if any. Module-level like DBOS itself: whoever shuts DBOS down ends it. */
+let systemPool;
+async function endSystemPool() {
+    const pool = systemPool;
+    systemPool = undefined;
+    await pool?.end().catch((err) => dbosLog.warn({ err }, "failed to close the DBOS system database pool"));
+}
 /** The executor whose deps the registered workflow uses. Set by launch(). */
 let activeExecutor;
 /** Setter (rather than `activeExecutor = this` inline) to avoid aliasing `this` to a variable. */
@@ -36,6 +91,27 @@ function setActiveExecutor(executor) {
  * `runInsideWorkflow`'s finally.
  */
 const cancellationReasons = new Map();
+/**
+ * The id a takeover gives the fork of workflow `from` under application `version`. Deterministic, so
+ * a takeover that forked but never recorded the fork finishes it rather than forking again; and
+ * descendants share their ancestor's id as a prefix, which isForkOf relies on.
+ */
+export function forkWorkflowId(from, version) {
+    return `${from}@${version}`;
+}
+/** Whether workflow `id` was forked from `from`, directly or through further forks (forkWorkflowId). */
+export function isForkOf(id, from) {
+    return id.startsWith(`${from}@`);
+}
+/**
+ * The DBOS workflow id that drives a run: `Run.executionWorkflowId` once the
+ * run's workflow was forked under a new application version, otherwise the
+ * run id (a run's first workflow is started with its run id).
+ */
+export async function workflowIdOf(db, runId) {
+    const row = await db.run.findUnique({ where: { id: runId }, select: { executionWorkflowId: true } });
+    return row?.executionWorkflowId ?? runId;
+}
 /**
  * Bind the engine's step boundary to a DBOS checkpointed step. Never
  * retried: a retry would re-spend.
@@ -44,23 +120,108 @@ const cancellationReasons = new Map();
  * out of the next step boundary. Translated here into a `RunCancelledError`
  * so executeRun's backstop records `cancelled` plus the operator's reason,
  * rather than `failed` plus an SDK-internal message.
+ *
+ * DBOS signals that another execution of this workflow recorded the step
+ * first by throwing `DBOSWorkflowConflictError` after this attempt's step
+ * body already ran. Translated into a `RunOwnershipLostError` (carrying the
+ * discarded step's usage, if any, for the log) so executeRun's backstop stops
+ * this attempt without writing the run; runInsideWorkflow turns it back into
+ * the SDK error.
+ *
+ * DBOS signals that a replay reached a step whose recorded name differs (the
+ * run was recorded by a release whose steps differ, e.g. after a takeover
+ * fork) by throwing `DBOSUnexpectedStepError`. Translated into a
+ * `StepReplayMismatchError`, which executeRun's backstop settles as `lost`
+ * (runInsideWorkflow does it for a mismatch at the load step, which is outside
+ * that backstop).
+ */
+export const dbosStep = (name, fn) => runDbosStep(name, fn);
+/**
+ * dbosStep for one run's workflow body. A cancellation of a workflow the run has moved off (its
+ * `executionWorkflowId` names a fork of it) is not a stop: recover() took the run over (forked it) and cancelled this,
+ * the forked-from workflow, while its owner was still alive. That attempt stops like one that lost
+ * a step's checkpoint (RunOwnershipLostError: no terminal write), so the fork, not a reasonless
+ * "cancelled" from the original, settles the run. takeOver records the fork's id before it
+ * cancels, so the original always reads the new id by the time it sees the cancel.
+ */
+export function dbosStepFor(db, runId) {
+    const supersedes = async (cancelledWorkflowId) => {
+        // A failed read falls back to an ordinary cancellation, as before this check existed.
+        const current = await workflowIdOf(db, runId).catch(() => cancelledWorkflowId);
+        return isForkOf(current, cancelledWorkflowId);
+    };
+    return (name, fn) => runDbosStep(name, fn, supersedes);
+}
+/**
+ * dbosStepFor behind the same per-step row read as inline runs (stopAwareStep), failing open. DBOS
+ * stops a workflow it was told to cancel, but stop() can cancel only the workflow the run names, and
+ * a takeover's fork is named only once recorded: a fork already running before that (or whose
+ * record failed) would otherwise keep paying for turns under a run stop() already made terminal.
+ * The same read fences out a taken-over original as soon as the run names its fork, before the
+ * takeover's cancel lands, and even when that cancel failed: it stops as an ownership loss, without
+ * a terminal write or the settle side effects (host checks, status comments) the fork owns.
  */
-const dbosStep = async (name, fn) => {
+export function stopAwareDbosStep(db, runId) {
+    return stopAwareStep(db, runId, dbosStepFor(db, runId), {
+        failOpen: true,
+        ownership: { workflowId: () => DBOS.workflowID, forkedFrom: isForkOf },
+    });
+}
+async function runDbosStep(name, fn, supersededBy) {
+    let ran;
     try {
-        return await DBOS.runStep(fn, { name, retriesAllowed: false });
+        return await DBOS.runStep(async () => {
+            const value = await fn();
+            ran = { value };
+            return value;
+        }, { name, retriesAllowed: false });
     }
     catch (err) {
         if (err instanceof DbosErrors.DBOSWorkflowCancelledError) {
+            if (await supersededBy?.(err.workflowID))
+                throw new RunOwnershipLostError(name, { cause: err });
             const reason = cancellationReasons.get(err.workflowID);
             throw new RunCancelledError(`Run cancelled: ${reason ?? "no reason given"}`);
         }
+        if (err instanceof DbosErrors.DBOSWorkflowConflictError) {
+            const discardedUsage = usageOf(ran?.value);
+            throw new RunOwnershipLostError(name, { cause: err, ...(discardedUsage ? { discardedUsage } : {}) });
+        }
+        if (err instanceof DbosErrors.DBOSUnexpectedStepError)
+            throw new StepReplayMismatchError(name, { cause: err });
         throw err;
     }
-};
+}
+/**
+ * The numeric `usage` fields of a step result, if it has any (an LLM turn's
+ * token counts and cost). Tool steps return a string and yield nothing.
+ */
+function usageOf(value) {
+    if (typeof value !== "object" || value === null || !("usage" in value))
+        return undefined;
+    const usage = value.usage;
+    if (typeof usage !== "object" || usage === null)
+        return undefined;
+    const numeric = Object.entries(usage).filter((entry) => typeof entry[1] === "number");
+    return numeric.length > 0 ? Object.fromEntries(numeric) : undefined;
+}
 /**
  * Registered once at module load — DBOS requires registration before
  * launch(). The body must be deterministic between steps; everything
  * non-deterministic in executeRun/engine is inside a runStep.
+ *
+ * Replay constraints. A run's workflow is replayed both by a same-version
+ * resume and, when recover() takes a waiting lead over after a deploy, by a
+ * fork that runs this body under the *new* release against steps the *old*
+ * release recorded (see decideTakeover). So:
+ * - Code between steps must stay deterministic and safe to repeat: it runs
+ *   again on every replay, now possibly under newer code.
+ * - Step names and their order must only change in ways a replay can still
+ *   follow; a replay that meets a different recorded name ends the run lost
+ *   with STEPS_CHANGED_REASON (executeRun's backstop, or runInsideWorkflow).
+ * - New code must accept step outputs recorded by the previous release (the
+ *   `turn:N:llm` TurnResult, tool results). A release that changes a recorded
+ *   shape keeps reading the old one, or a takeover fails at that step (lost).
  */
 const runWorkflow = DBOS.registerWorkflow(async (runId) => {
     const self = activeExecutor;
@@ -112,15 +273,24 @@ export class DbosExecutor {
         }
         setActiveExecutor(this);
         if (!DBOS.isInitialized()) {
-            DBOS.setConfig({
-                name: "wardby",
-                systemDatabaseUrl: this.config.systemDatabaseUrl,
-                systemDatabaseSchemaName: this.config.schemaName,
-                executorID: this.id,
-                runAdminServer: false,
-                logLevel: "warn",
-            });
-            await DBOS.launch();
+            const launchConfig = dbosLaunchConfig(this.config, this.id, databaseAuthFromEnv());
+            systemPool = launchConfig.systemDatabasePool;
+            try {
+                DBOS.setConfig(launchConfig);
+                await DBOS.launch();
+            }
+            catch (err) {
+                // DBOS.launch can throw after it initialized (registering the application version, starting
+                // event receivers). Shut that half-launched DBOS down so the next launch() starts clean, then
+                // end the pool either way: the next launch() makes a new one.
+                if (activeExecutor === this)
+                    setActiveExecutor(undefined);
+                if (DBOS.isInitialized()) {
+                    await DBOS.shutdown().catch((shutdownErr) => dbosLog.warn({ err: shutdownErr }, "failed to shut down DBOS after a failed launch"));
+                }
+                await endSystemPool();
+                throw err;
+            }
         }
         this.launched = true;
         dbosLog.info({ executorId: DBOS.executorID }, "DBOS executor launched");
@@ -129,9 +299,18 @@ export class DbosExecutor {
         if (!this.launched)
             return;
         this.launched = false;
-        if (activeExecutor === this)
-            setActiveExecutor(undefined);
-        await DBOS.shutdown({ workflowCompletionTimeoutMS: 5_000 });
+        try {
+            await DBOS.shutdown({ workflowCompletionTimeoutMS: 5_000 });
+        }
+        finally {
+            // Only now: shutdown first lets the queue runner finish the dispatch it is in, so a workflow
+            // the queue claimed just before (a takeover's fork, an adoption) still starts. Without its
+            // executor its body would throw, and DBOS could record that as the workflow's outcome: the
+            // next process would then mark a run failed that was only waiting to be taken over.
+            if (activeExecutor === this)
+                setActiveExecutor(undefined);
+            await endSystemPool();
+        }
     }
     /** Where a run this executor marks failed files its self-defect (no-op without a configured issue tracker). */
     get selfDefects() {
@@ -169,6 +348,15 @@ export class DbosExecutor {
             // the same failure — dispatchRun's own `.catch` around `start()` calls
             // it too — which is harmless precisely because it is conditional and
             // idempotent: the second call matches zero rows.
+            //
+            // A workflow that recover() took over (forked, then cancelled) no longer drives the run: the
+            // fork does, and settles it. Its rejection is the takeover's cancel, not the run's failure.
+            if ((await workflowIdOf(this.db, runId).catch(() => runId)) !== runId) {
+                // The error's name only: this path adds no message text to any output.
+                const error = err instanceof Error ? err.name : typeof err;
+                dbosLog.info({ runId, error }, "the run's original workflow ended after a takeover; its fork drives the run");
+                return;
+            }
             await markRunFailedFromExecutorError(this.db, runId, err, this.selfDefects);
             throw err;
         }
@@ -180,11 +368,43 @@ export class DbosExecutor {
         if (!this.launched)
             await this.launch();
         dbosLog.info({ runId, reason }, "cancelling durable run");
+        // The terminal write comes first and is conditional (the first writer wins), so the run is
+        // cancelled with the caller's reason even when its workflow runs in another process (where the
+        // reason map below is empty) or has no workflow at all yet (cancelWorkflow then matches
+        // nothing). The workflow's next step then throws, and its own terminal write matches no row.
+        // The cancel is still attempted when that settle throws (a host or issue update failing after
+        // the write) or the database is unreachable: a cancelled row whose workflow kept going would
+        // keep spending. The settle's error, if any, is the one rethrown.
+        let settleFailed;
+        try {
+            const row = await this.db.run.findUnique({ where: { id: runId }, select: { status: true } });
+            if (row && (row.status === "pending" || row.status === "running")) {
+                await failNativeRun({ runId, db: this.db, providers: this.providers, ...nativeRunIntegrations(this.providers, this.db) }, new RunCancelledError(reason ?? "The run was cancelled."));
+            }
+        }
+        catch (err) {
+            settleFailed = { err };
+        }
         // Recorded before the cancel so the workflow's next step boundary can
         // find it: DBOS only reports "cancelled", never why.
-        if (reason)
+        // Keyed by both ids: dbosStep looks it up by the cancelled workflow's id, which is the forked
+        // id for a forked run. A failed read targets the run id, the workflow of any run never forked.
+        const workflowId = await workflowIdOf(this.db, runId).catch(() => runId);
+        if (reason) {
             cancellationReasons.set(runId, reason);
-        await DBOS.cancelWorkflow(runId);
+            cancellationReasons.set(workflowId, reason);
+        }
+        try {
+            await DBOS.cancelWorkflow(workflowId);
+        }
+        catch (err) {
+            if (!settleFailed)
+                throw err;
+            // The error's name only: this path adds no message text to any output.
+            dbosLog.warn({ runId, error: err instanceof Error ? err.name : typeof err }, "could not cancel the workflow");
+        }
+        if (settleFailed)
+            throw settleFailed.err;
     }
     /**
      * Answer the reconciler for a stale run. Never relaunches: `resume`
@@ -213,13 +433,12 @@ export class DbosExecutor {
                 // DBOS gates recovery and dequeue on application_version. A workflow
                 // recorded under a different version is never dequeued here, so
                 // resuming it would be an unbounded loop that holds the run
-                // `running` forever: give up at once and let the reconciler reap it.
+                // `running` forever. Instead it is forked under this version when it
+                // is only waiting on durable children, and otherwise given up at once
+                // so the reconciler reaps it (takeOver).
                 const mismatch = versionMismatchReason(status?.applicationVersion, DBOS.applicationVersion);
-                if (mismatch) {
-                    this.resumeAttempts.delete(handle.runId);
-                    dbosLog.warn({ runId: handle.runId, workflowVersion: status?.applicationVersion }, mismatch);
-                    return { state: "lost", reason: mismatch };
-                }
+                if (mismatch)
+                    return this.takeOver(handle, mismatch, status?.applicationVersion);
                 // Belt and braces for every other reason a resume can fail to take
                 // (an id we can't dequeue for, a queue that never drains): bound the
                 // number of adoptions of one run so the reconciler stops re-driving
@@ -253,6 +472,56 @@ export class DbosExecutor {
                 return { state: "lost", reason: decision.reason };
         }
     }
+    /**
+     * A workflow recorded under another application version: fork it under this one when it is only
+     * waiting on durable children (decideTakeover), otherwise answer lost with the mismatch. The
+     * fork counts as an adoption attempt, so repeated failed takeovers of one run end lost too.
+     */
+    async takeOver(handle, mismatch, workflowVersion) {
+        const steps = (await DBOS.listWorkflowSteps(handle.id)) ?? [];
+        const children = await this.db.run.findMany({
+            where: { parentRunId: handle.runId, status: { in: [...DRIVABLE] } },
+            select: { status: true, nativeExecutionMode: true, agent: { select: { kind: true } } },
+        });
+        const decision = decideTakeover(steps.map((s) => ({ functionID: s.functionID, name: s.name, output: s.output })), children.map((c) => ({ status: c.status, kind: c.agent.kind, nativeExecutionMode: c.nativeExecutionMode })), mismatch);
+        if (decision.action === "lost") {
+            this.resumeAttempts.delete(handle.runId);
+            dbosLog.warn({ runId: handle.runId, workflowVersion }, mismatch);
+            return { state: "lost", reason: decision.reason };
+        }
+        const attempts = this.resumeAttempts.get(handle.runId) ?? 0;
+        if (shouldGiveUpAdoption(attempts)) {
+            this.resumeAttempts.delete(handle.runId);
+            dbosLog.warn({ runId: handle.runId, attempts }, "giving up on taking over durable run");
+            return { state: "lost", reason: ADOPTION_EXHAUSTED_REASON };
+        }
+        this.resumeAttempts.set(handle.runId, attempts + 1);
+        // Deterministic, so a takeover that forked but never recorded the fork (the run write below
+        // failed, or the process died) is finished by the next pass rather than forked a second time:
+        // two workflows driving one run would each spend on every turn. DBOS inserts a fork's row with
+        // a plain INSERT (bulkForkWorkflows in the SDK's system_database.js), so a repeated id would
+        // fail on the key: look first, and treat a lost race the same way.
+        const forkedId = forkWorkflowId(handle.id, DBOS.applicationVersion);
+        if (!(await DBOS.getWorkflowStatus(forkedId))) {
+            try {
+                await DBOS.forkWorkflow(handle.id, decision.startStep, {
+                    newWorkflowID: forkedId,
+                    applicationVersion: DBOS.applicationVersion,
+                });
+            }
+            catch (err) {
+                if (!(await DBOS.getWorkflowStatus(forkedId)))
+                    throw err;
+            }
+        }
+        const forked = DBOS.retrieveWorkflow(forkedId);
+        await this.db.run.update({ where: { id: handle.runId }, data: { executionWorkflowId: forkedId } });
+        // So nothing can dequeue the original again (it stays recorded as forked from).
+        await DBOS.cancelWorkflow(handle.id).catch((err) => dbosLog.warn({ err, runId: handle.runId }, "could not cancel the forked-from workflow"));
+        dbosLog.info({ runId: handle.runId, from: handle.id, to: forkedId, startStep: decision.startStep, workflowVersion }, "run taken over under this application version");
+        void forked.getResult().catch((err) => dbosLog.error({ err, runId: handle.runId }, "taken-over run failed"));
+        return { state: "active" };
+    }
     /** Workflow body. Public only so the module-level registration can reach it. */
     async runInsideWorkflow(runId) {
         const beat = () => this.db.run.update({ where: { id: runId }, data: { heartbeatAt: new Date() } }).catch(() => {
@@ -261,11 +530,35 @@ export class DbosExecutor {
         await beat();
         const timer = setInterval(() => void beat(), this.heartbeatIntervalMs);
         try {
-            await executeRun(runId, this.providers, this.db, undefined, dbosStep);
+            await executeRun(runId, this.providers, this.db, undefined, stopAwareDbosStep(this.db, runId));
+        }
+        catch (err) {
+            // An ownership loss leaves the workflow body as the SDK's own conflict error, so DBOS takes
+            // its duplicate-execution path: it records nothing and parks this attempt until the winner
+            // records the outcome. Any other error escaping here would be recorded as the workflow's
+            // outcome while the winner still runs. The runner already logged the loss. A forked-from
+            // workflow's own cancellation (dbosStepFor) goes back as itself: DBOS parks that too.
+            if (err instanceof RunOwnershipLostError) {
+                throw err.cause instanceof DbosErrors.DBOSWorkflowConflictError ||
+                    err.cause instanceof DbosErrors.DBOSWorkflowCancelledError
+                    ? err.cause
+                    : new DbosErrors.DBOSWorkflowConflictError(DBOS.workflowID ?? runId);
+            }
+            // A replay met a step the recorded run does not have at the load step, ahead of executeRun's
+            // backstop (which settles a mismatch in any later step itself).
+            if (err instanceof StepReplayMismatchError) {
+                dbosLog.warn({ err, runId }, "replay no longer matches the recorded run");
+                // The same settle as executeRun's backstop: lost (first writer wins), its children stopped.
+                await loseNativeRun({ runId, db: this.db, providers: this.providers, ...nativeRunIntegrations(this.providers, this.db) }, STEPS_CHANGED_REASON);
+                return;
+            }
+            throw err;
         }
         finally {
             clearInterval(timer);
             cancellationReasons.delete(runId);
+            if (DBOS.workflowID)
+                cancellationReasons.delete(DBOS.workflowID);
         }
     }
 }

package/dist/providers/executor/deferred.d.ts

@@ -0,0 +1,44 @@
+/**
+ * An Executor for a process that must never start or stop runs itself: the
+ * native sandbox gateway (docs/native-sandbox.md). Sandbox workers can reach
+ * the gateway, so it holds no Docker socket, cluster credential, or in-process
+ * engine; a child it dispatches is only recorded (Run.startDeferredAt) and a
+ * stop it needs only requested (Run.stopRequestedAt). The scheduler leader
+ * carries both out through the server's own executor (`drainDeferredRuns`).
+ *
+ * Coding dispatch is recorded unresolved (`defersCodingResolution`): the worker image,
+ * service declaration and knowledge files are resolved by the process that starts the run
+ * (the drain's `beforeStart` hook), not here.
+ */
+import type { PrismaClient } from "#prisma";
+import type { Executor } from "./types.js";
+export type DeferredExecutorDb = Pick<PrismaClient, "run">;
+export declare class DeferredExecutor implements Executor {
+    private readonly db;
+    readonly defersCodingResolution = true;
+    constructor(db: DeferredExecutorDb);
+    start(runId: string): Promise<void>;
+    stop(runId: string, reason?: string): Promise<void>;
+}
+export interface DrainDeferredRunsOptions {
+    db: Pick<PrismaClient, "run">;
+    executor: Executor;
+    /** Where a start failure is recorded; defaults to marking the run failed with the error. */
+    onStartFailed?: (runId: string, err: unknown) => Promise<void>;
+    /**
+     * Runs after a row is claimed and before `executor.start`. Resolves inputs the deferring process
+     * could not; false means the hook ended the run itself so it must not start. A throw is handled
+     * like a start failure.
+     */
+    beforeStart?: (runId: string) => Promise<boolean>;
+    limit?: number;
+}
+/**
+ * Starts runs a deferred executor dispatched, and stops runs it asked to stop, through the
+ * server's executor. Called on the scheduler leader's tick. Each row is claimed by clearing its
+ * marker in a conditional update first, so two drains never start or stop the same run twice.
+ */
+export declare function drainDeferredRuns(options: DrainDeferredRunsOptions): Promise<{
+    started: number;
+    stopped: number;
+}>;

package/dist/providers/executor/deferred.js

@@ -0,0 +1,100 @@
+/**
+ * An Executor for a process that must never start or stop runs itself: the
+ * native sandbox gateway (docs/native-sandbox.md). Sandbox workers can reach
+ * the gateway, so it holds no Docker socket, cluster credential, or in-process
+ * engine; a child it dispatches is only recorded (Run.startDeferredAt) and a
+ * stop it needs only requested (Run.stopRequestedAt). The scheduler leader
+ * carries both out through the server's own executor (`drainDeferredRuns`).
+ *
+ * Coding dispatch is recorded unresolved (`defersCodingResolution`): the worker image,
+ * service declaration and knowledge files are resolved by the process that starts the run
+ * (the drain's `beforeStart` hook), not here.
+ */
+import { logger } from "../../core/logger.js";
+const deferredLog = logger.child({ module: "deferred-executor" });
+export class DeferredExecutor {
+    db;
+    defersCodingResolution = true;
+    constructor(db) {
+        this.db = db;
+    }
+    async start(runId) {
+        await this.db.run.updateMany({ where: { id: runId, status: "pending" }, data: { startDeferredAt: new Date() } });
+    }
+    async stop(runId, reason) {
+        // Never started: nothing runs it, so it can simply end here.
+        const cancelled = await this.db.run.updateMany({
+            where: { id: runId, status: "pending", startDeferredAt: { not: null } },
+            data: {
+                status: "cancelled",
+                error: reason ?? "Cancelled before it started.",
+                finishedAt: new Date(),
+                startDeferredAt: null,
+            },
+        });
+        if (cancelled.count > 0)
+            return;
+        await this.db.run.updateMany({
+            where: { id: runId, status: { in: ["pending", "running"] } },
+            data: { stopRequestedAt: new Date(), stopRequestReason: reason ?? null },
+        });
+    }
+}
+/**
+ * Starts runs a deferred executor dispatched, and stops runs it asked to stop, through the
+ * server's executor. Called on the scheduler leader's tick. Each row is claimed by clearing its
+ * marker in a conditional update first, so two drains never start or stop the same run twice.
+ */
+export async function drainDeferredRuns(options) {
+    const { db, executor } = options;
+    const limit = options.limit ?? 50;
+    let started = 0;
+    let stopped = 0;
+    const toStart = await db.run.findMany({
+        where: { startDeferredAt: { not: null }, status: "pending" },
+        orderBy: { startDeferredAt: "asc" },
+        take: limit,
+        select: { id: true },
+    });
+    for (const { id } of toStart) {
+        const claimed = await db.run.updateMany({
+            where: { id, startDeferredAt: { not: null }, status: "pending" },
+            data: { startDeferredAt: null },
+        });
+        if (claimed.count === 0)
+            continue;
+        started += 1;
+        // Not awaited: an in-process executor's start runs the whole run.
+        void (async () => {
+            if (options.beforeStart && !(await options.beforeStart(id)))
+                return;
+            await executor.start(id);
+        })().catch(async (err) => {
+            deferredLog.error({ err, runId: id }, "deferred run failed to start");
+            await (options.onStartFailed?.(id, err) ??
+                db.run.updateMany({
+                    where: { id, status: { in: ["pending", "running"] } },
+                    data: { status: "failed", error: err instanceof Error ? err.message : String(err), finishedAt: new Date() },
+                }));
+        });
+    }
+    const toStop = await db.run.findMany({
+        where: { stopRequestedAt: { not: null } },
+        orderBy: { stopRequestedAt: "asc" },
+        take: limit,
+        select: { id: true, stopRequestReason: true, status: true },
+    });
+    for (const row of toStop) {
+        const claimed = await db.run.updateMany({
+            where: { id: row.id, stopRequestedAt: { not: null } },
+            data: { stopRequestedAt: null, stopRequestReason: null },
+        });
+        if (claimed.count === 0 || !["pending", "running"].includes(row.status))
+            continue;
+        stopped += 1;
+        await executor.stop(row.id, row.stopRequestReason ?? undefined).catch((err) => {
+            deferredLog.warn({ err, runId: row.id }, "deferred stop request failed");
+        });
+    }
+    return { started, stopped };
+}

package/dist/providers/executor/in-process.d.ts

@@ -13,5 +13,13 @@ export declare class InProcessExecutor implements Executor {
     private readonly heartbeatIntervalMs;
     constructor(providers: NativeRunProviders, db?: RunnerDb, heartbeatIntervalMs?: number);
     start(runId: string): Promise<void>;
-    stop(_runId: string, _reason?: string): Promise<void>;
+    /**
+     * Records the run cancelled (only while it is still pending or running: the first terminal write
+     * wins) and settles it like any finished run (its review check, status comments, notifications).
+     * The engine itself is stopped cooperatively: executeRun's inline step runner reads the run's row
+     * before every step (stopAwareStep), so the process executing it, this one or another, stops it
+     * at the next step boundary, and a run that has not started yet never does. A step already in
+     * flight (one model turn or tool call) finishes first; its spend is still recorded.
+     */
+    stop(runId: string, reason?: string): Promise<void>;
 }

package/dist/providers/executor/in-process.js

@@ -5,7 +5,7 @@
  * heartbeat and recovers the run to `lost` rather than leaving it stuck
  * `running` forever.
  */
-import { executeRun } from "../../core/runner.js";
+import { executeRun, failNativeRun, nativeRunIntegrations, RunCancelledError, } from "../../core/runner.js";
 import { prisma as defaultDb } from "../../core/db.js";
 import { HEARTBEAT_INTERVAL_MS } from "../../core/timing.js";
 export class InProcessExecutor {
@@ -31,8 +31,18 @@ export class InProcessExecutor {
             clearInterval(timer);
         }
     }
-    async stop(_runId, _reason) {
-        // Native engine cancellation remains cooperative until Engine accepts an
-        // AbortSignal. The task state still records the caller's request.
+    /**
+     * Records the run cancelled (only while it is still pending or running: the first terminal write
+     * wins) and settles it like any finished run (its review check, status comments, notifications).
+     * The engine itself is stopped cooperatively: executeRun's inline step runner reads the run's row
+     * before every step (stopAwareStep), so the process executing it, this one or another, stops it
+     * at the next step boundary, and a run that has not started yet never does. A step already in
+     * flight (one model turn or tool call) finishes first; its spend is still recorded.
+     */
+    async stop(runId, reason) {
+        const run = await this.db.run.findUnique({ where: { id: runId }, select: { status: true } });
+        if (!run || (run.status !== "pending" && run.status !== "running"))
+            return;
+        await failNativeRun({ runId, db: this.db, providers: this.providers, ...nativeRunIntegrations(this.providers, this.db) }, new RunCancelledError(reason ?? "The run was cancelled."));
     }
 }

package/dist/providers/executor/linked-issue-reader.d.ts

@@ -0,0 +1,16 @@
+import type { IssueTrackerRegistry } from "../issue-tracker/types.js";
+import type { ReviewHostRegistry } from "../review-host/types.js";
+import type { LinkedIssueRef } from "./container.js";
+/**
+ * The executor's readLinkedIssue: a GitHub issue through the review host's
+ * issue read (the App's installation token for the run's repository), a Jira
+ * issue through the configured tracker. Null when the number is a pull
+ * request, the issue is missing, or no host or tracker is configured.
+ */
+export declare function linkedIssueReader(deps: {
+    hosts: ReviewHostRegistry;
+    issueTrackers: IssueTrackerRegistry;
+}): (ref: LinkedIssueRef) => Promise<{
+    title: string;
+    body: string;
+} | null>;