@wardby/cli 0.1.0

package/dist/core/reconciler.js

@@ -0,0 +1,152 @@
+/**
+ * Orphan recovery: an executor-managed run whose process died mid-execution
+ * stops beating `Run.heartbeatAt` (or never got a chance to create the
+ * `Run` at all). Any instance (not gated by the scheduler lease — recovery
+ * shouldn't wait on leadership) periodically transitions such runs to a
+ * terminal `lost` state.
+ *
+ * Scoped by `executionManaged`, not trigger. Scheduled, MCP, and webhook
+ * runs are detached and managed; an attended `wardby run` is not. Coding
+ * jobs with persisted handles are delegated to the executor's recovery
+ * path, which must query/stop/collect and must never relaunch.
+ *
+ * Two states get reclaimed:
+ *  - `running` past the heartbeat timeout (the executor died mid-run). The
+ *    `heartbeatAt IS NULL` arm is a defensive fallback for a future
+ *    executor that doesn't beat before flipping to `running` —
+ *    `InProcessExecutor` always beats first, so a live scheduled run is
+ *    never actually in `running` with a null heartbeat.
+ *  - `pending` past the timeout (the process died between `claimDueRun`
+ *    committing the `Run` and the executor ever starting it — or the
+ *    executor's `start()` call itself failed before persisting anything).
+ *    Queued coding runs (`CodingRun.queuedAt` set) are excluded; the coding
+ *    queue owns their timeout.
+ *
+ * The conditional `updateMany` (`WHERE ... AND status IN (...) AND ...`) is
+ * what makes two concurrent reconcilers safe: whichever transaction's
+ * UPDATE commits first flips the row out of the matched status, so the
+ * second one's WHERE simply matches zero rows — double-reconcile is a
+ * no-op, not a race.
+ */
+import { HEARTBEAT_TIMEOUT_MS, RECONCILE_INTERVAL_MS } from "./timing.js";
+import { prisma as defaultDb } from "./db.js";
+import { logger } from "./logger.js";
+const reconcilerLog = logger.child({ module: "reconciler" });
+/** Runs one reconciliation pass. Returns the number of runs marked `lost`. */
+export async function reconcileOnce(db, now = new Date(), heartbeatTimeoutMs = HEARTBEAT_TIMEOUT_MS, executor) {
+    const cutoff = new Date(now.getTime() - heartbeatTimeoutMs);
+    const stale = {
+        executionManaged: true,
+        OR: [
+            {
+                status: "running",
+                OR: [{ heartbeatAt: { lt: cutoff } }, { heartbeatAt: null, startedAt: { lt: cutoff } }],
+            },
+            {
+                status: "pending",
+                startedAt: { lt: cutoff },
+                // A coding run waiting for a concurrency slot is pending by design;
+                // drainCodingQueue times those out (coding_queue_timeout) instead.
+                OR: [{ codingRun: { is: null } }, { codingRun: { is: { queuedAt: null } } }],
+            },
+        ],
+    };
+    const candidates = await db.run.findMany({
+        where: stale,
+        include: {
+            agent: { select: { kind: true } },
+            codingRun: { select: { jobBackend: true, jobHandle: true } },
+        },
+    });
+    let lost = 0;
+    for (const run of candidates) {
+        let reason = `Orphaned: no heartbeat/progress since before ${cutoff.toISOString()}.`;
+        if (run.agent.kind === "coding" && run.codingRun?.jobHandle) {
+            if (!executor?.recover || !run.codingRun.jobBackend)
+                continue;
+            let recovered;
+            try {
+                recovered = await executor.recover({
+                    runId: run.id,
+                    backend: run.codingRun.jobBackend,
+                    id: run.codingRun.jobHandle,
+                });
+            }
+            catch (err) {
+                reconcilerLog.error({ err, runId: run.id }, "managed run recovery failed");
+                continue;
+            }
+            if (recovered.state === "active") {
+                await db.run.updateMany({
+                    where: { id: run.id, executionManaged: true, status: { in: ["pending", "running"] } },
+                    data: { heartbeatAt: now },
+                });
+                continue;
+            }
+            if (recovered.state === "terminal")
+                continue;
+            reason = recovered.reason ?? "Managed coding job was lost after stop and collection attempts.";
+        }
+        else if (run.agent.kind === "coding") {
+            reason = "Managed coding job became stale before its launcher handle was persisted; it was not relaunched.";
+        }
+        else if (run.executionBackend && executor?.recover) {
+            // Claim before asking. For a `running` row this is a real CAS: the
+            // winner's UPDATE refreshes heartbeatAt, which takes the row out of
+            // the stale set, so a concurrent reconciler's WHERE matches zero rows
+            // and only one instance calls recover() this pass.
+            //
+            // For a `pending` row it de-duplicates nothing across passes: `pending`
+            // is matched on startedAt, not heartbeatAt, so the row stays stale and
+            // every later pass calls recover() again. That is accepted rather than
+            // fixed: DBOS's dequeue is atomic and `resumeWorkflow` is idempotent
+            // (re-enqueueing an already-enqueued workflow does not run it twice),
+            // and DbosExecutor bounds the repeats — after MAX_RESUME_ATTEMPTS
+            // adoptions of one run it reports `lost` and the run gets reaped.
+            const claimed = await db.run.updateMany({
+                where: { id: run.id, ...stale },
+                data: { heartbeatAt: now },
+            });
+            if (claimed.count === 0)
+                continue;
+            let recovered;
+            try {
+                recovered = await executor.recover({ runId: run.id, backend: run.executionBackend, id: run.id });
+            }
+            catch (err) {
+                reconcilerLog.error({ err, runId: run.id }, "durable run recovery failed");
+                continue;
+            }
+            if (recovered.state === "active" || recovered.state === "terminal")
+                continue;
+            reason = recovered.reason ?? "Durable backend reported the run lost.";
+            const result = await db.run.updateMany({
+                where: { id: run.id, executionManaged: true, status: { in: ["pending", "running"] } },
+                data: { status: "lost", error: reason, finishedAt: now },
+            });
+            lost += result.count;
+            continue;
+        }
+        const result = await db.run.updateMany({
+            where: { id: run.id, ...stale },
+            data: { status: "lost", error: reason, finishedAt: now },
+        });
+        lost += result.count;
+    }
+    return lost;
+}
+export function startReconciler(options = {}) {
+    const db = options.db ?? defaultDb;
+    const intervalMs = options.intervalMs ?? RECONCILE_INTERVAL_MS;
+    const heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? HEARTBEAT_TIMEOUT_MS;
+    const timer = setInterval(() => {
+        reconcileOnce(db, new Date(), heartbeatTimeoutMs, options.executor).catch((err) => {
+            reconcilerLog.error({ err }, "reconcile pass failed");
+        });
+    }, intervalMs);
+    return {
+        stop() {
+            clearInterval(timer);
+        },
+    };
+}

package/dist/core/runner.d.ts

@@ -0,0 +1,87 @@
+/**
+ * The runner. Split into `createRun` (persist a pending Run for an agent)
+ * and `executeRun` (drive an existing Run to a terminal state) so Phase 2's
+ * scheduler can create the Run itself (inside its claim transaction) and
+ * hand the id to an `Executor`, which is what actually calls `executeRun`.
+ * `runAgent` is the convenience that does both in one call, and is what the
+ * CLI's `wardby run` uses directly (an attended, foreground command doesn't
+ * need the executor's heartbeat/reconciler durability — only unattended
+ * scheduled runs do).
+ *
+ * Phase 3: `executeRun` is now a thin wrapper. It loads the agent and its
+ * attached tools, builds the `EngineRunContext` (wiring `runSandboxTool` to
+ * the Zod-in-sandbox validation + WASM sandbox), calls the configured
+ * `Engine`, and persists the `EngineResult`. Every budget decision
+ * (pre-flight refuse, cumulative pre-turn gate, mid-stream cutoff, wind-
+ * down) lives inside the engine now — there is exactly one place that
+ * reasons about cost, not one here plus one in the engine.
+ */
+import type { PrismaClient, Run, RunTrigger } from "@prisma/client";
+import type { ProviderRegistry } from "../providers/index.js";
+import { type StepRunner } from "../providers/engine/types.js";
+import type { Executor } from "../providers/executor/types.js";
+/**
+ * The subset of the Prisma client the runner touches — mockable in tests.
+ * Includes `codingRun`/`task`/`webhook`/`$transaction`/`$queryRaw` so this
+ * also satisfies `dispatch.ts`'s `DispatchDb`, needed for dispatching a
+ * coding-kind sub-agent from inside a running native turn loop.
+ */
+export type RunnerDb = Pick<PrismaClient, "agent" | "run" | "agentTool" | "agentSecret" | "agentDatastore" | "budgetGroup" | "agentSubAgent" | "codingRun" | "task" | "webhook" | "$transaction" | "$queryRaw">;
+/**
+ * Raised when a run's step boundary observed a cancellation (DbosExecutor's
+ * `stop`). `executeRun`'s backstop persists these as `cancelled` with the
+ * operator's reason, rather than `failed` with an executor-internal message.
+ */
+export declare class RunCancelledError extends Error {
+    constructor(message: string);
+}
+/**
+ * Slack past the child's own queue wait + container deadline before the
+ * parent gives up: the executor needs a moment after a deadline kill to
+ * write the terminal row, and the queue timeout only fires on a drain tick.
+ */
+export declare const CODING_CHILD_WAIT_GRACE_SEC = 60;
+export type CodingChildWaitOutcome = {
+    kind: "terminal";
+    run: Run;
+} | {
+    kind: "timed_out";
+} | {
+    kind: "parent_cancelled";
+};
+export interface WaitForCodingChildOptions {
+    db: Pick<RunnerDb, "run" | "task">;
+    executor: Pick<Executor, "stop">;
+    childRunId: string;
+    parentRunId: string;
+    /** Give up after this long; the child is stopped so it never runs on detached. */
+    boundMs: number;
+    initialPollMs?: number;
+    maxPollMs?: number;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
+/**
+ * Polls a dispatched coding child until it reaches a terminal status. Needed
+ * because a queued child (every concurrency slot taken) makes Executor.start
+ * resolve while the run is still pending; drainCodingQueue runs it later.
+ *
+ * Parent cancellation is cooperative everywhere in the runner (DbosExecutor
+ * only interrupts at a step boundary, and this wait runs inside one tool
+ * step), so it is observed from the database instead: the parent Run left
+ * pending/running (reaped as lost, or finished elsewhere), or an MCP task
+ * for the parent run was cancelled. Either way, and on timeout, the child is
+ * stopped rather than left to spend on behalf of a parent that stopped
+ * waiting for it.
+ */
+export declare function waitForCodingChild(options: WaitForCodingChildOptions): Promise<CodingChildWaitOutcome>;
+/** Persists a new pending Run for the named agent. Throws if the agent is unknown. */
+export declare function createRun(db: RunnerDb, agentName: string, trigger?: RunTrigger): Promise<Run>;
+/** Drives an existing Run (created by `createRun` or the scheduler) to a terminal state. */
+export declare function executeRun(runId: string, providers: Pick<ProviderRegistry, "llm" | "engine" | "datastore" | "secrets" | "memory"> & {
+    executor?: ProviderRegistry["executor"];
+}, db?: RunnerDb, onText?: (delta: string) => void, step?: StepRunner): Promise<Run>;
+/** Convenience: create + execute a manual run in one call (what the CLI's `wardby run` uses). */
+export declare function runAgent(agentName: string, providers: Pick<ProviderRegistry, "llm" | "engine" | "datastore" | "secrets" | "memory"> & {
+    executor?: ProviderRegistry["executor"];
+}, db?: RunnerDb, onText?: (delta: string) => void): Promise<Run>;