@godspeedai/cognate-runtime-bun 0.4.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@godspeedai/cognate-runtime-bun",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Bun reference runtime: composes kernel, durable store, runs, continuations, commitments, artifacts, and cognition behind RuntimeService.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,18 +22,18 @@
     ".": "./src/index.ts"
   },
   "dependencies": {
-    "@godspeedai/cognate-artifacts": "^0.4.0",
-    "@godspeedai/cognate-capabilities": "^0.4.0",
-    "@godspeedai/cognate-cognitive-api": "^0.4.0",
-    "@godspeedai/cognate-continuations": "^0.4.0",
-    "@godspeedai/cognate-events": "^0.4.0",
-    "@godspeedai/cognate-kernel-api": "^0.4.0",
-    "@godspeedai/cognate-kernel-cordis": "^0.4.0",
-    "@godspeedai/cognate-projections": "^0.4.0",
-    "@godspeedai/cognate-runtime-api": "^0.4.0",
-    "@godspeedai/cognate-store-memory": "^0.4.0",
-    "@godspeedai/cognate-store-postgres": "^0.4.0",
-    "@godspeedai/cognate-store-sqlite": "^0.4.0"
+    "@godspeedai/cognate-artifacts": "^0.6.0",
+    "@godspeedai/cognate-capabilities": "^0.6.0",
+    "@godspeedai/cognate-cognitive-api": "^0.6.0",
+    "@godspeedai/cognate-continuations": "^0.6.0",
+    "@godspeedai/cognate-events": "^0.6.0",
+    "@godspeedai/cognate-kernel-api": "^0.6.0",
+    "@godspeedai/cognate-kernel-cordis": "^0.6.0",
+    "@godspeedai/cognate-projections": "^0.6.0",
+    "@godspeedai/cognate-runtime-api": "^0.6.0",
+    "@godspeedai/cognate-store-memory": "^0.6.0",
+    "@godspeedai/cognate-store-postgres": "^0.6.0",
+    "@godspeedai/cognate-store-sqlite": "^0.6.0"
   },
   "devDependencies": {
     "@godspeedai/cognate-cognitive-memory": "workspace:*"

package/src/core.ts

@@ -28,6 +28,8 @@ export interface Core {
   readonly outboxFor: (event: PendingEvent) => readonly OutboxIntent[];
   /** Abort controllers of executions in progress, by run id (cancellation aborts `RunContext.signal`). */
   readonly controllers: Map<string, AbortController>;
+  /** Stops cooperative execution without cancelling its durable objective. */
+  readonly shutdownSignal: AbortSignal;
   readonly remoteOffers: RemoteOfferRegistry;
   /** Serialize and (re-)run a run's execution; replay-safe and idempotent. */
   execute(runId: string): Promise<void>;

package/src/executor.ts

@@ -1,7 +1,7 @@
 // Durable run execution by deterministic replay (ADR-0011, spec §11, §12).
 // Each execution re-reads the run stream, re-enters the agent from the top, and answers every
 // RunContext call that already has a recorded result from the stream instead of repeating it.
-import { ConcurrencyError, fingerprint, IdempotencyMismatchError, type Json, type NewEvent, type StoredEvent } from "@godspeedai/cognate-events";
+import { ConcurrencyError, fingerprint, IdempotencyMismatchError, type Json, type NewEvent, StaleRunOwnerError, type StoredEvent } from "@godspeedai/cognate-events";
 import type { Principal, SemanticRef } from "@godspeedai/cognate-kernel-api";
 import { applyStateUpdate, type ArtifactInput, ContinuationClosedError, FrontendCapabilityError, foldState, InvocationFailedError, type RemoteCapabilityOffer, type RunContext, StatePatchError, type WaitSpec } from "@godspeedai/cognate-runtime-api";
 import type { Core, RecallResultJson } from "./core.ts";
@@ -14,9 +14,36 @@ class Halt {
 
 export type ExecutionResult = "done" | "retry";
 
-export async function executeOnce(core: Core, runId: string): Promise<ExecutionResult> {
+/** The fenced run-write ownership a worker holds for one execution pass
+ * (docs/contracts/run-write-ownership.md). The credential fences every run-scoped append; renew
+ * refreshes the lease and reports whether ownership was lost. */
+export interface RunExecutionOwnership {
+  readonly ownerId: string;
+  readonly generation: number;
+  readonly ambiguousSteps: "reexecute" | "wait";
+  readonly renew: () => Promise<boolean>;
+}
+
+export async function executeOnce(core: Core, runId: string, ownership?: RunExecutionOwnership): Promise<ExecutionResult> {
+  // Register before the first store await so cancellation and shutdown can reach this attempt
+  // while its initial snapshot is being read.
+  const controller = new AbortController();
+  core.controllers.set(runId, controller);
+  const stopForShutdown = () => controller.abort(new Error("runtime closed"));
+  if (core.shutdownSignal.aborted) stopForShutdown();
+  else core.shutdownSignal.addEventListener("abort", stopForShutdown, { once: true });
+  try {
+    return await executeRegistered(core, runId, controller, ownership);
+  } finally {
+    core.shutdownSignal.removeEventListener("abort", stopForShutdown);
+    if (core.controllers.get(runId) === controller) core.controllers.delete(runId);
+  }
+}
+
+async function executeRegistered(core: Core, runId: string, controller: AbortController, ownership?: RunExecutionOwnership): Promise<ExecutionResult> {
   const streamId = runStream(runId);
   const events = await core.store.readStream(streamId);
+  if (controller.signal.aborted) return "done";
   const view = foldRun(runId, events);
   if (!view || isTerminal(view)) return "done";
   const agent = core.agents.get(view.agent);
@@ -31,8 +58,6 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
   }
   const find = (stepId: string, eventType?: string) => recorded.get(stepId)?.find((e) => eventType === undefined || e.eventType === eventType);
 
-  const controller = new AbortController();
-  core.controllers.set(runId, controller);
   let halted: Halt | undefined;
   const halt = (retry: boolean): never => {
     halted ??= new Halt(retry);
@@ -66,13 +91,23 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
       try {
         // Pre-assign the id so hooks can derive intents that commit atomically with the event (spec §13.1).
         const pending = { ...envelope, eventId: crypto.randomUUID(), streamId, recordedAt: envelope.occurredAt };
-        const outcome = await core.store.commit({ appends: [{ streamId, expectedVersion: version, events: [{ ...envelope, eventId: pending.eventId }] }], outbox: core.outboxFor(pending) });
+        const outcome = await core.store.commit({
+          appends: [{
+            streamId,
+            expectedVersion: version,
+            events: [{ ...envelope, eventId: pending.eventId }],
+            ...(ownership ? { runOwnership: { runId, ownerId: ownership.ownerId, generation: ownership.generation } } : {}),
+          }],
+          outbox: core.outboxFor(pending),
+        });
         const stored = outcome.events[0]!;
         version += 1;
         lastEventId = stored.eventId;
         return stored;
       } catch (error) {
         if (error instanceof ConcurrencyError) halt(true);
+        // Ownership was reclaimed or released: this worker is stale and must never retry the pass.
+        if (error instanceof StaleRunOwnerError) halt(false);
         throw error;
       }
     });
@@ -134,17 +169,49 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
       // recorded event) but are a distinct event type, so the completion check must name it explicitly.
       let done = find(stepId, "run.invocation");
       if (!done) {
+        // Refresh the ownership lease before any external effect: a lost lease must be discovered
+        // before dispatch, not after (docs/contracts/run-write-ownership.md).
+        if (ownership && !(await ownership.renew())) halt(false);
+        const priorProgress = (recorded.get(stepId) ?? []).filter((e) => e.eventType === "run.invocation.progress");
+        // A previous owner's generation started this invocation and never recorded its outcome.
+        // Under `ambiguousSteps: "wait"` the recovering owner does not re-perform the effect: it
+        // records the ambiguity and waits for explicit reconciliation.
+        if (ownership && ownership.ambiguousSteps === "wait") {
+          const priorGenerations = priorProgress
+            .map((e) => field<number>(meta(e), "ownerGeneration"))
+            .filter((generation): generation is number => typeof generation === "number" && generation < ownership.generation);
+          if (priorGenerations.length > 0) {
+            const priorGeneration = Math.max(...priorGenerations);
+            const ambiguousStepId = `${stepId}:ambiguous`;
+            if (!find(ambiguousStepId, "run.invocation.ambiguous")) {
+              await append({ eventType: "run.invocation.ambiguous", payload: { capability, priorGeneration }, metadata: { stepId: ambiguousStepId, capability, priorGeneration } });
+            }
+            const resolution = await context.waitFor(`ambiguous:${stepId}`, {
+              type: "ambiguous-invocation",
+              condition: { kind: "ambiguous_invocation", capability, priorGeneration },
+              payload: { capability, stepId, priorGeneration },
+            });
+            const reconciled = (resolution ?? null) as { readonly result?: Json; readonly failure?: { readonly kind: string; readonly message?: string } } | null;
+            if (reconciled?.failure) throw new InvocationFailedError(capability, reconciled.failure);
+            return reconciled?.result ?? null;
+          }
+        }
         // A crash between a progress call and the final run.invocation event leaves partial progress
         // recorded for this stepId with no completion; re-executing calls the handler again under a
         // fresh attempt number so consumers can discard the earlier partial attempt.
-        const priorAttempts = (recorded.get(stepId) ?? [])
-          .filter((e) => e.eventType === "run.invocation.progress")
-          .map((e) => field<number>(meta(e), "attempt") ?? 0);
+        const priorAttempts = priorProgress.map((e) => field<number>(meta(e), "attempt") ?? 0);
         const attempt = priorAttempts.length ? Math.max(...priorAttempts) + 1 : 0;
         let seq = 0;
         const progress = async (payload: Json): Promise<void> => {
           guard();
-          await append({ eventType: "run.invocation.progress", payload: toJson(payload), metadata: { stepId, capability, attempt, seq: seq++ } });
+          await append({
+            eventType: "run.invocation.progress",
+            payload: toJson(payload),
+            metadata: {
+              stepId, capability, attempt, seq: seq++,
+              ...(ownership ? { ownerGeneration: ownership.generation } : {}),
+            },
+          });
         };
         const call = (approval?: { readonly approvalId: string }) =>
           core.invoker.invoke({
@@ -343,12 +410,14 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
                   payload: toJson(update),
                   metadata: { scope: "thread", threadId: view.correlationId, source: "agent", actor: context.actor.id, stepId: `shared-state.update:${id}` },
                 }],
+                ...(ownership ? { runOwnership: { runId, ownerId: ownership.ownerId, generation: ownership.generation } } : {}),
               }],
               result: { state, version: events.length + 1 },
             });
             return outcome.result as { readonly state: Json; readonly version: number };
           } catch (error) {
             if (error instanceof ConcurrencyError) continue;
+            if (error instanceof StaleRunOwnerError) halt(false);
             if (error instanceof IdempotencyMismatchError) throw new Error(`thread state update ${id} was replayed with different input`);
             throw error;
           }
@@ -403,6 +472,7 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
     if (!agent) outcome = { ok: false, error: new Error(`agent ${view.agent} is not registered`) };
     else {
       try {
+        guard();
         outcome = { ok: true, value: toJson(await agent.run(field(started.payload, "input"), context)) };
       } catch (error) {
         outcome = { ok: false, error };
@@ -417,7 +487,5 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
   } catch (error) {
     if (error instanceof Halt) return error.retry ? "retry" : "done";
     throw error;
-  } finally {
-    if (core.controllers.get(runId) === controller) core.controllers.delete(runId);
   }
 }

package/src/index.ts

@@ -70,9 +70,26 @@ export interface RuntimeOptions {
   readonly execution?: {
     readonly maxConcurrentRuns?: number;
     /** Required above capacity one. Returning [] explicitly declares independent eligibility.
-     * Trusted resource identities derived from frozen run inputs, never model-granted authority.
-     * Runs sharing a key serialize in this process; this is not a distributed lock or OS isolation. */
-    readonly resourceKeys?: (run: RunView) => readonly string[];
+  readonly execution?: {
+    readonly maxConcurrentRuns?: number;
+    /** Required above capacity one. Returning [] explicitly declares independent eligibility.
+     * This trusted, side-effect-free policy may asynchronously resolve canonical resource identities
+     * from authoritative application metadata. Its signal is aborted when the run is cancelled or the runtime closes.
+     * Runs sharing a key serialize in this process; this is not distributed ownership or OS isolation. */
+    readonly resourceKeys?: (run: RunView, signal?: AbortSignal) => readonly string[] | Promise<readonly string[]>;
+    /** Fenced run-write ownership tuning (docs/contracts/run-write-ownership.md). Claims happen
+     * automatically whenever the store implements the extension; SQLite/memory stay legacy. */
+    readonly runOwnership?: {
+      /** Stable worker identity for claims; defaults to a per-runtime random id. */
+      readonly ownerId?: string;
+      /** Lease duration in milliseconds; default 30_000. Expiry is reclaim eligibility, not
+       * ownership loss — a matching generation stays writable until someone reclaims. */
+      readonly leaseMs?: number;
+      /** What a recovering owner does with a previous generation's started-but-unrecorded
+       * invocation: "reexecute" (historical behavior, default) or "wait" — record an ambiguity
+       * and require explicit reconciliation before the external effect may repeat. */
+      readonly ambiguousSteps?: "reexecute" | "wait";
+    };
   };
   readonly system?: Principal;
 }

package/src/runtime.ts

@@ -1,15 +1,17 @@
 // Composition root: store, kernel, continuations, artifacts, projections, cognition, and the background loop.
+import { randomUUID } from "node:crypto";
 import { ArtifactService } from "@godspeedai/cognate-artifacts";
 import { ContinuationService } from "@godspeedai/cognate-continuations";
-import { type DurableStore, type Json, NotFoundError, OutboxDispatcher } from "@godspeedai/cognate-events";
+import { type DurableStore, type Json, NotFoundError, OutboxDispatcher, RunOwnershipHeldError } from "@godspeedai/cognate-events";
 import { createCordisKernel } from "@godspeedai/cognate-kernel-cordis";
 import type { Principal } from "@godspeedai/cognate-kernel-api";
+import type { RunView } from "@godspeedai/cognate-runtime-api";
 import { ProjectionRunner } from "@godspeedai/cognate-projections";
 import { openPostgresStore, resolvePg0Url } from "@godspeedai/cognate-store-postgres";
 import { openSqliteStore } from "@godspeedai/cognate-store-sqlite";
 import { COGNITION_PROJECTION, cognitionProjection, recallFor } from "./cognition.ts";
 import type { Core } from "./core.ts";
-import { executeOnce } from "./executor.ts";
+import { executeOnce, type RunExecutionOwnership } from "./executor.ts";
 import { ExecutionSlots } from "./execution-slots.ts";
 import { foldRun, isTerminal, RESUME_HANDLER, RUN_EXECUTE_TOPIC, RUNS, runStream } from "./fold.ts";
 import type { Runtime, RuntimeOptions } from "./index.ts";
@@ -33,8 +35,15 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     }
   }
   const slots=new ExecutionSlots(capacity);
+  // Fenced run-write ownership worker identity (docs/contracts/run-write-ownership.md). Per
+  // runtime instance: a restarted process reclaims with a fresh generation.
+  const ownerId=options.execution?.runOwnership?.ownerId ?? `worker-${crypto.randomUUID()}`;
+  const leaseMs=options.execution?.runOwnership?.leaseMs ?? 30_000;
+  // Recovery can enqueue many runs at once. Bound authoritative eligibility lookups with the
+  // same primitive/capacity so metadata resolution cannot fan out without limit.
+  const eligibilitySlots = new ExecutionSlots(capacity);
   // Run and resume topics share one pre-claim budget. ExecutionSlots separately enforces
-  // run-local/resource exclusion for every execution path, including direct recovery.
+  // run-local/resource exclusion for every execution path.
   let admittedDeliveries = 0;
   const deliveryAdmission = { tryAcquire(): (() => void) | undefined {
     if (admittedDeliveries >= capacity) return undefined;
@@ -70,8 +79,81 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
   const continuations = new ContinuationService(store, { policy: { authorize: () => ({ allow: true, reason: "authorized by runtime service" }) }, delivery: DELIVERY, deliveryConcurrency: capacity, deliveryAdmission });
 
   let closed = false;
+  const shutdown = new AbortController();
+  const executionControllers = new Map<string, AbortController>();
+  const acquireExecutionSlot = async (runId: string, view: RunView): Promise<(() => void) | undefined> => {
+    const resolveKeys = options.execution?.resourceKeys;
+    const controller = new AbortController();
+    const signal = controller.signal;
+    executionControllers.set(runId, controller);
+    const abortForShutdown = () => controller.abort(new Error("runtime closed"));
+    if (shutdown.signal.aborted) abortForShutdown();
+    else shutdown.signal.addEventListener("abort", abortForShutdown, { once: true });
+    let onAbort!: () => void;
+    const aborted = new Promise<{ readonly kind: "aborted" }>((resolve) => {
+      onAbort = () => resolve({ kind: "aborted" });
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort, { once: true });
+    });
+    let releaseEligibility: (() => void) | undefined;
+    try {
+      const current = foldRun(runId, await store.readStream(runStream(runId)));
+      if (closed || signal.aborted || !current || isTerminal(current)) return undefined;
+      let keys: readonly string[] = [];
+      if (resolveKeys) {
+        // A cancelled waiter may still be in this semaphore queue. Attach a release handler now so
+        // its eventual permit is returned even when the abort race lets execute() stop waiting.
+        const acquisition = eligibilitySlots.acquire([]).then((release) => {
+          if (closed || signal.aborted) { release(); return undefined; }
+          releaseEligibility = release;
+          return release;
+        });
+        const admitted = await Promise.race([acquisition, aborted]);
+        if (!admitted || closed || signal.aborted) return undefined;
+        // A callback that ignores cancellation may settle later, but its eventual rejection remains
+        // observed by this handler after the abort race has returned.
+        const lookup = Promise.resolve().then(async () => {
+          if (closed || signal.aborted) return { kind: "aborted" as const };
+          try { return { kind: "keys" as const, keys: await resolveKeys(view, signal) }; }
+          catch (error) { return { kind: "error" as const, error }; }
+        });
+        const result = await Promise.race([lookup, aborted]);
+        if (result.kind === "aborted") {
+          // Cancellation releases the caller immediately, but an uncancellable policy callback
+          // still occupies a real lookup slot until it settles. The outcome union observes late
+          // rejection, and the queued release preserves the configured concurrency bound.
+          const release = releaseEligibility;
+          releaseEligibility = undefined;
+          void lookup.then(() => release?.());
+          return undefined;
+        }
+        if (result.kind === "error") throw result.error;
+        if (!Array.isArray(result.keys)) throw new Error("execution.resourceKeys must resolve to an array");
+        keys = result.keys;
+      }
+      // Keep the run controller registered until both policy and execution-slot admission finish.
+      // If cancellation wins while queued, a later grant is immediately returned by this handler.
+      const acquisition = slots.acquire(keys).then((release) => {
+        if (closed || signal.aborted) { release(); return undefined; }
+        return release;
+      });
+      return await Promise.race([acquisition, aborted.then(() => undefined)]);
+    } finally {
+      signal.removeEventListener("abort", onAbort);
+      shutdown.signal.removeEventListener("abort", abortForShutdown);
+      releaseEligibility?.();
+      if (executionControllers.get(runId) === controller) executionControllers.delete(runId);
+    }
+  };
   const chains = new Map<string, Promise<void>>();
   const pending = new Set<string>();
+  // A runtime epoch gives recovery attempts fresh durable intent IDs after prior intents have
+  // already been delivered. Existing pending run.execute intents retain their persisted backoff.
+  const recoveryEpoch = randomUUID();
+  let recoverySequence = 0;
+  const recoveryReconciliationIntervalMs = 250;
+  let nextRecoveryReconciliationAt = 0;
+  const recoveryOutboxByRun = new Map<string, string>();
   const deliveries = new Set<Promise<void>>();
   const trackDelivery=(job:Promise<unknown>):void=>{
     const tracked=job.then(()=>{},()=>{}).finally(()=>deliveries.delete(tracked));
@@ -81,9 +163,34 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     const next = (chains.get(runId) ?? Promise.resolve()).then(async () => {
       const view=foldRun(runId,await store.readStream(runStream(runId)));
       if(closed || !view || isTerminal(view))return;
-      const release=await slots.acquire(options.execution?.resourceKeys?.(view) ?? []);
+      const release=await acquireExecutionSlot(runId, view);
+      if (!release) return;
       try{
-        for (let attempt = 0; attempt < 5 && !closed; attempt++) if ((await executeOnce(core, runId)) === "done") return;
+        if (closed) return;
+        // Fenced run-write ownership (docs/contracts/run-write-ownership.md): claim before
+        // executing. A held lease means another live worker is executing this run — back off;
+        // its completion, suspension or death releases the run for a later pass to reclaim.
+        let ownership: RunExecutionOwnership | undefined;
+        if (store.claimRunOwnership) {
+          try {
+            const lease = await store.claimRunOwnership({ runId, ownerId, now: Date.now(), leaseMs });
+            ownership = {
+              ownerId,
+              generation: lease.generation,
+              ambiguousSteps: options.execution?.runOwnership?.ambiguousSteps ?? "reexecute",
+              renew: () => store.renewRunOwnership!({ runId, ownerId, generation: lease.generation, now: Date.now(), leaseMs }).then((ok) => ok === true),
+            };
+          } catch (error) {
+            if (error instanceof RunOwnershipHeldError) return;
+            throw error;
+          }
+        }
+        try{
+          for (let attempt = 0; attempt < 5 && !closed; attempt++) if ((await executeOnce(core, runId, ownership)) === "done") return;
+        }finally{
+          // Best-effort release: a reclaimed or already-released row simply refuses.
+          if (ownership) await store.releaseRunOwnership!({ runId, ownerId, generation: ownership.generation }).catch(() => {});
+        }
       }finally{release();}
     });
     const tracked = next.finally(() => {
@@ -104,7 +211,8 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     scopeId: kernel.root.id,
     recall: recallFor(store, options.cognition),
     outboxFor: options.outboxFor ?? (() => []),
-    controllers: new Map(),
+    controllers: executionControllers,
+    shutdownSignal: shutdown.signal,
     remoteOffers: new RemoteOfferRegistry(),
     execute,
     kick: () => kick(),
@@ -122,14 +230,58 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     admission: deliveryAdmission,
     backoff: DELIVERY,
     deliver: async (item) => {
-      await execute((item.payload as { runId: string }).runId);
-      return { executed: true };
+      const runId = (item.payload as { runId: string }).runId;
+      try {
+        await execute(runId);
+        return { executed: true };
+      } finally {
+        // The store retains pending retry/poison state; this map only correlates this process's
+        // currently delivered recovery intent and must not retain failed or exhausted attempts.
+        if (recoveryOutboxByRun.get(runId) === item.id) recoveryOutboxByRun.delete(runId);
+      }
     },
   });
 
+  async function persistRecoveryIntents(): Promise<void> {
+    // Reconcile only when a new recovery candidate exists. A tracked intent already has a
+    // durable retry schedule; rescanning the complete pending topic every timer tick is needless.
+    if (pending.size === 0) return;
+    const now = performance.now();
+    if (now < nextRecoveryReconciliationAt) return;
+    // Set the deadline before awaiting: a slow or failing store query must not hot-loop scans.
+    nextRecoveryReconciliationAt = now + recoveryReconciliationIntervalMs;
+    const pendingItems = await store.listOutbox({ topic: RUN_EXECUTE_TOPIC, status: "pending" });
+    const pendingItemIds = new Set(pendingItems.map((item) => item.id));
+    for (const [runId, id] of recoveryOutboxByRun) {
+      if (pendingItemIds.has(id)) pending.delete(runId);
+      else recoveryOutboxByRun.delete(runId);
+    }
+    const candidates = [...pending].filter((runId) => !recoveryOutboxByRun.has(runId)).slice(0, 100);
+    if (candidates.length === 0) return;
+    const pendingRunItems = new Map<string, string>();
+    for (const item of pendingItems) {
+      const runId = (item.payload as { runId?: unknown } | null)?.runId;
+      if (typeof runId === "string" && !pendingRunItems.has(runId)) pendingRunItems.set(runId, item.id);
+    }
+    const intents: { id: string; topic: string; payload: Json }[] = [];
+    const selected = new Map<string, string>();
+    for (const runId of candidates) {
+      const existing = pendingRunItems.get(runId);
+      const id = existing ?? `${RUN_EXECUTE_TOPIC}:recovery:${recoveryEpoch}:${++recoverySequence}:${runId}`;
+      selected.set(runId, id);
+      if (!existing) intents.push({ id, topic: RUN_EXECUTE_TOPIC, payload: { runId } });
+    }
+    if (intents.length > 0) await store.commit({ appends: [], outbox: intents });
+    for (const [runId, id] of selected) {
+      recoveryOutboxByRun.set(runId, id);
+      pending.delete(runId);
+    }
+  }
+
   let resumeFirst = false;
   async function tick(): Promise<void> {
     for (const record of await continuations.expireDue()) pending.add(record.runId);
+    await persistRecoveryIntents();
     // Dispatchers reserve a bounded set of slots across overlapping drains. A new runnable
     // arrival can fill a free slot while an earlier long-running delivery is still executing.
     // Rotate priority: even an empty topic briefly reserves capacity while its claim is
@@ -142,9 +294,6 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
       trackDelivery(continuations.dispatchResumes());
     }
     resumeFirst = !resumeFirst;
-    const recover=[...pending];
-    for(const runId of recover)pending.delete(runId);
-    for(const runId of recover)trackDelivery(execute(runId));
   }
 
   let ticking: Promise<void> | null = null;
@@ -190,7 +339,10 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     if (view.status === "running") pending.add(id);
     for (const continuationId of view.waitingOn) {
       const record = await continuations.get(continuationId);
-      if (record && !["OPEN", "BLOCKED", "RESUMED"].includes(record.status)) pending.add(id);
+      // A resume delivery can be acknowledged during shutdown before run.resumed is appended.
+      // Re-enter closed waits, including RESUMED, even when their outbox receipt is delivered.
+      // The run-local execution chain and recorded wait result make duplicate delivery harmless.
+      if (record && !["OPEN", "BLOCKED"].includes(record.status)) pending.add(id);
     }
   }
 
@@ -381,6 +533,7 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     async close() {
       if (closed) return;
       closed = true;
+      shutdown.abort();
       clearInterval(timer);
       if (approvalTimer) clearInterval(approvalTimer);
       await ticking;