@godspeedai/cognate-runtime-bun 0.3.0 → 0.5.0

package/README.md

@@ -124,12 +124,14 @@ kept current automatically), `cognition` (index completed runs and recall them v
 
 ## Limitations
 
-- **One process, one writer.** All runs live in the process that opened the store; executions
-  within a tick run sequentially, so a slow agent delays others. To scale out, split runtimes per
-  store file.
+- **One runtime execution owner per store.** PostgreSQL supports multiple durable-store writers;
+  that does not establish distributed run ownership. Run execution defaults to serial. Opt-in
+  `execution.maxConcurrentRuns` admits bounded independent runs in one process and requires a
+  trusted `execution.resourceKeys` eligibility policy above one. Shared resource keys serialize;
+  callers must canonicalize aliases. Do not start competing runtimes against the same runnable work.
 - **Steps are at-least-once.** A step that was in flight during a crash re-executes after restart.
-  Keep steps idempotent, and put irreversible external calls behind a `ctx.waitFor` continuation so
-  they happen once, resolved deliberately.
+  Keep steps idempotent, and use native commitment receipts and explicit reconciliation for external
+  actions. A continuation or permission gate alone does not establish exactly-once physical effects.
 - **Polling, not pushing, inside the process.** Background work (run execution, resume delivery,
   continuation expiry) runs on a loop every `tickMs`.
 - **Erasure has copies.** `runtime.erase` removes an event's payload from the log, derived views,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@godspeedai/cognate-runtime-bun",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Bun reference runtime: composes kernel, durable store, runs, continuations, commitments, artifacts, and cognition behind RuntimeService.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,18 +22,18 @@
     ".": "./src/index.ts"
   },
   "dependencies": {
-    "@godspeedai/cognate-artifacts": "^0.3.0",
-    "@godspeedai/cognate-capabilities": "^0.3.0",
-    "@godspeedai/cognate-cognitive-api": "^0.3.0",
-    "@godspeedai/cognate-continuations": "^0.3.0",
-    "@godspeedai/cognate-events": "^0.3.0",
-    "@godspeedai/cognate-kernel-api": "^0.3.0",
-    "@godspeedai/cognate-kernel-cordis": "^0.3.0",
-    "@godspeedai/cognate-projections": "^0.3.0",
-    "@godspeedai/cognate-runtime-api": "^0.3.0",
-    "@godspeedai/cognate-store-memory": "^0.3.0",
-    "@godspeedai/cognate-store-postgres": "^0.3.0",
-    "@godspeedai/cognate-store-sqlite": "^0.3.0"
+    "@godspeedai/cognate-artifacts": "^0.5.0",
+    "@godspeedai/cognate-capabilities": "^0.5.0",
+    "@godspeedai/cognate-cognitive-api": "^0.5.0",
+    "@godspeedai/cognate-continuations": "^0.5.0",
+    "@godspeedai/cognate-events": "^0.5.0",
+    "@godspeedai/cognate-kernel-api": "^0.5.0",
+    "@godspeedai/cognate-kernel-cordis": "^0.5.0",
+    "@godspeedai/cognate-projections": "^0.5.0",
+    "@godspeedai/cognate-runtime-api": "^0.5.0",
+    "@godspeedai/cognate-store-memory": "^0.5.0",
+    "@godspeedai/cognate-store-postgres": "^0.5.0",
+    "@godspeedai/cognate-store-sqlite": "^0.5.0"
   },
   "devDependencies": {
     "@godspeedai/cognate-cognitive-memory": "workspace:*"

package/src/core.ts

@@ -28,6 +28,8 @@ export interface Core {
   readonly outboxFor: (event: PendingEvent) => readonly OutboxIntent[];
   /** Abort controllers of executions in progress, by run id (cancellation aborts `RunContext.signal`). */
   readonly controllers: Map<string, AbortController>;
+  /** Stops cooperative execution without cancelling its durable objective. */
+  readonly shutdownSignal: AbortSignal;
   readonly remoteOffers: RemoteOfferRegistry;
   /** Serialize and (re-)run a run's execution; replay-safe and idempotent. */
   execute(runId: string): Promise<void>;

package/src/execution-slots.ts

@@ -0,0 +1,37 @@
+/** Admission inside the native executor, shared by start/recovery/resume paths. This is a
+ * process-local limit and resource exclusion, not durable ownership of an execution. */
+export class ExecutionSlots {
+  private active = 0;
+  private readonly held = new Set<string>();
+  private readonly waiting: {keys: readonly string[]; resolve: (release: () => void) => void}[] = [];
+  constructor(readonly capacity: number) {}
+
+  acquire(raw: readonly string[]): Promise<() => void> {
+    if (!Array.isArray(raw) || raw.length > 100 || raw.some(key=>typeof key!=="string" || !key || key.length>500)) {
+      return Promise.reject(new Error("execution.resourceKeys must return at most 100 nonempty resource identities"));
+    }
+    return new Promise(resolve=>{
+      this.waiting.push({keys:[...new Set(raw)].sort(),resolve});
+      this.pump();
+    });
+  }
+  private pump(): void {
+    const earlier=new Set<string>();
+    for(let i=0;i<this.waiting.length && this.active<this.capacity;){
+      const item=this.waiting[i]!;
+      if(item.keys.some(key=>this.held.has(key) || earlier.has(key))){
+        for(const key of item.keys)earlier.add(key);
+        i++;continue;
+      }
+      this.waiting.splice(i,1);this.active++;
+      for(const key of item.keys)this.held.add(key);
+      let released=false;
+      item.resolve(()=>{
+        if(released)return;
+        released=true;this.active--;
+        for(const key of item.keys)this.held.delete(key);
+        this.pump();
+      });
+    }
+  }
+}

package/src/executor.ts

@@ -15,8 +15,25 @@ class Halt {
 export type ExecutionResult = "done" | "retry";
 
 export async function executeOnce(core: Core, runId: string): Promise<ExecutionResult> {
+  // Register before the first store await so cancellation and shutdown can reach this attempt
+  // while its initial snapshot is being read.
+  const controller = new AbortController();
+  core.controllers.set(runId, controller);
+  const stopForShutdown = () => controller.abort(new Error("runtime closed"));
+  if (core.shutdownSignal.aborted) stopForShutdown();
+  else core.shutdownSignal.addEventListener("abort", stopForShutdown, { once: true });
+  try {
+    return await executeRegistered(core, runId, controller);
+  } finally {
+    core.shutdownSignal.removeEventListener("abort", stopForShutdown);
+    if (core.controllers.get(runId) === controller) core.controllers.delete(runId);
+  }
+}
+
+async function executeRegistered(core: Core, runId: string, controller: AbortController): Promise<ExecutionResult> {
   const streamId = runStream(runId);
   const events = await core.store.readStream(streamId);
+  if (controller.signal.aborted) return "done";
   const view = foldRun(runId, events);
   if (!view || isTerminal(view)) return "done";
   const agent = core.agents.get(view.agent);
@@ -31,8 +48,6 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
   }
   const find = (stepId: string, eventType?: string) => recorded.get(stepId)?.find((e) => eventType === undefined || e.eventType === eventType);
 
-  const controller = new AbortController();
-  core.controllers.set(runId, controller);
   let halted: Halt | undefined;
   const halt = (retry: boolean): never => {
     halted ??= new Halt(retry);
@@ -403,6 +418,7 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
     if (!agent) outcome = { ok: false, error: new Error(`agent ${view.agent} is not registered`) };
     else {
       try {
+        guard();
         outcome = { ok: true, value: toJson(await agent.run(field(started.payload, "input"), context)) };
       } catch (error) {
         outcome = { ok: false, error };
@@ -417,7 +433,5 @@ export async function executeOnce(core: Core, runId: string): Promise<ExecutionR
   } catch (error) {
     if (error instanceof Halt) return error.retry ? "retry" : "done";
     throw error;
-  } finally {
-    if (core.controllers.get(runId) === controller) core.controllers.delete(runId);
   }
 }

package/src/index.ts

@@ -2,7 +2,7 @@ import type { CognitiveIndex, Embedder, IndexDocument } from "@godspeedai/cognat
 import type { DurableStore, Json, NewEvent, OutboxIntent, StoredEvent } from "@godspeedai/cognate-events";
 import type { Component, Kernel, Policy, Principal } from "@godspeedai/cognate-kernel-api";
 import type { Projection } from "@godspeedai/cognate-projections";
-import type { AgentDefinition, Caller, RuntimeService } from "@godspeedai/cognate-runtime-api";
+import type { AgentDefinition, Caller, RuntimeService, RunView } from "@godspeedai/cognate-runtime-api";
 import type { GovernanceOptions } from "./governed-operation.ts";
 import { startRuntime } from "./runtime.ts";
 
@@ -66,6 +66,15 @@ export interface RuntimeOptions {
   readonly outboxFor?: (event: PendingEvent) => readonly OutboxIntent[];
   /** Background work loop interval (run execution, resume delivery, expiry). Default 10 ms. */
   readonly tickMs?: number;
+  /** Explicit single-process execution admission; defaults to serial compatibility. */
+  readonly execution?: {
+    readonly maxConcurrentRuns?: number;
+    /** Required above capacity one. Returning [] explicitly declares independent eligibility.
+     * This trusted, side-effect-free policy may asynchronously resolve canonical resource identities
+     * from authoritative application metadata. Its signal is aborted when the run is cancelled or the runtime closes.
+     * Runs sharing a key serialize in this process; this is not distributed ownership or OS isolation. */
+    readonly resourceKeys?: (run: RunView, signal?: AbortSignal) => readonly string[] | Promise<readonly string[]>;
+  };
   readonly system?: Principal;
 }
 

package/src/runtime.ts

@@ -1,15 +1,18 @@
 // Composition root: store, kernel, continuations, artifacts, projections, cognition, and the background loop.
+import { randomUUID } from "node:crypto";
 import { ArtifactService } from "@godspeedai/cognate-artifacts";
 import { ContinuationService } from "@godspeedai/cognate-continuations";
 import { type DurableStore, type Json, NotFoundError, OutboxDispatcher } from "@godspeedai/cognate-events";
 import { createCordisKernel } from "@godspeedai/cognate-kernel-cordis";
 import type { Principal } from "@godspeedai/cognate-kernel-api";
+import type { RunView } from "@godspeedai/cognate-runtime-api";
 import { ProjectionRunner } from "@godspeedai/cognate-projections";
 import { openPostgresStore, resolvePg0Url } from "@godspeedai/cognate-store-postgres";
 import { openSqliteStore } from "@godspeedai/cognate-store-sqlite";
 import { COGNITION_PROJECTION, cognitionProjection, recallFor } from "./cognition.ts";
 import type { Core } from "./core.ts";
 import { executeOnce } from "./executor.ts";
+import { ExecutionSlots } from "./execution-slots.ts";
 import { foldRun, isTerminal, RESUME_HANDLER, RUN_EXECUTE_TOPIC, RUNS, runStream } from "./fold.ts";
 import type { Runtime, RuntimeOptions } from "./index.ts";
 import { inspectCorrelation } from "./inspect.ts";
@@ -22,6 +25,28 @@ const DELIVERY = { baseMs: 50, maxMs: 5_000, maxAttempts: 10 };
 const TOPICS = [RUN_EXECUTE_TOPIC, "continuation.resume"];
 
 export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
+  const capacity=options.execution?.maxConcurrentRuns ?? 1;
+  if(!Number.isInteger(capacity) || capacity<1 || capacity>64)throw new Error("execution.maxConcurrentRuns must be an integer between 1 and 64");
+  if(capacity>1 && typeof options.execution?.resourceKeys!=="function")throw new Error("Concurrent execution requires an explicit execution.resourceKeys eligibility policy");
+  if(capacity>1 && typeof options.store!=="string" && "store" in options.store){
+    const injected=options.store.store;
+    if(typeof injected.claimOutboxLeased!=="function" || typeof injected.renewOutboxLease!=="function" || typeof injected.settleOutboxLease!=="function"){
+      throw new Error("Concurrent execution requires a store with complete fenced outbox lease support");
+    }
+  }
+  const slots=new ExecutionSlots(capacity);
+  // Recovery can enqueue many runs at once. Bound authoritative eligibility lookups with the
+  // same primitive/capacity so metadata resolution cannot fan out without limit.
+  const eligibilitySlots = new ExecutionSlots(capacity);
+  // Run and resume topics share one pre-claim budget. ExecutionSlots separately enforces
+  // run-local/resource exclusion for every execution path.
+  let admittedDeliveries = 0;
+  const deliveryAdmission = { tryAcquire(): (() => void) | undefined {
+    if (admittedDeliveries >= capacity) return undefined;
+    admittedDeliveries++;
+    let released = false;
+    return () => { if (!released) { released = true; admittedDeliveries--; } };
+  } };
   // The runtime closes only stores it opened itself; an injected store is owned by its creator.
   // (The PostgreSQL store's close() ends its pool and LISTEN connection, so one call covers both.)
   const config = options.store;
@@ -47,14 +72,99 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
   await invoker.attach(kernel, options.components ?? []);
 
   // Authorization happens in the service against `actions`; the continuation layer trusts its caller.
-  const continuations = new ContinuationService(store, { policy: { authorize: () => ({ allow: true, reason: "authorized by runtime service" }) }, delivery: DELIVERY });
+  const continuations = new ContinuationService(store, { policy: { authorize: () => ({ allow: true, reason: "authorized by runtime service" }) }, delivery: DELIVERY, deliveryConcurrency: capacity, deliveryAdmission });
 
   let closed = false;
+  const shutdown = new AbortController();
+  const executionControllers = new Map<string, AbortController>();
+  const acquireExecutionSlot = async (runId: string, view: RunView): Promise<(() => void) | undefined> => {
+    const resolveKeys = options.execution?.resourceKeys;
+    const controller = new AbortController();
+    const signal = controller.signal;
+    executionControllers.set(runId, controller);
+    const abortForShutdown = () => controller.abort(new Error("runtime closed"));
+    if (shutdown.signal.aborted) abortForShutdown();
+    else shutdown.signal.addEventListener("abort", abortForShutdown, { once: true });
+    let onAbort!: () => void;
+    const aborted = new Promise<{ readonly kind: "aborted" }>((resolve) => {
+      onAbort = () => resolve({ kind: "aborted" });
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort, { once: true });
+    });
+    let releaseEligibility: (() => void) | undefined;
+    try {
+      const current = foldRun(runId, await store.readStream(runStream(runId)));
+      if (closed || signal.aborted || !current || isTerminal(current)) return undefined;
+      let keys: readonly string[] = [];
+      if (resolveKeys) {
+        // A cancelled waiter may still be in this semaphore queue. Attach a release handler now so
+        // its eventual permit is returned even when the abort race lets execute() stop waiting.
+        const acquisition = eligibilitySlots.acquire([]).then((release) => {
+          if (closed || signal.aborted) { release(); return undefined; }
+          releaseEligibility = release;
+          return release;
+        });
+        const admitted = await Promise.race([acquisition, aborted]);
+        if (!admitted || closed || signal.aborted) return undefined;
+        // A callback that ignores cancellation may settle later, but its eventual rejection remains
+        // observed by this handler after the abort race has returned.
+        const lookup = Promise.resolve().then(async () => {
+          if (closed || signal.aborted) return { kind: "aborted" as const };
+          try { return { kind: "keys" as const, keys: await resolveKeys(view, signal) }; }
+          catch (error) { return { kind: "error" as const, error }; }
+        });
+        const result = await Promise.race([lookup, aborted]);
+        if (result.kind === "aborted") {
+          // Cancellation releases the caller immediately, but an uncancellable policy callback
+          // still occupies a real lookup slot until it settles. The outcome union observes late
+          // rejection, and the queued release preserves the configured concurrency bound.
+          const release = releaseEligibility;
+          releaseEligibility = undefined;
+          void lookup.then(() => release?.());
+          return undefined;
+        }
+        if (result.kind === "error") throw result.error;
+        if (!Array.isArray(result.keys)) throw new Error("execution.resourceKeys must resolve to an array");
+        keys = result.keys;
+      }
+      // Keep the run controller registered until both policy and execution-slot admission finish.
+      // If cancellation wins while queued, a later grant is immediately returned by this handler.
+      const acquisition = slots.acquire(keys).then((release) => {
+        if (closed || signal.aborted) { release(); return undefined; }
+        return release;
+      });
+      return await Promise.race([acquisition, aborted.then(() => undefined)]);
+    } finally {
+      signal.removeEventListener("abort", onAbort);
+      shutdown.signal.removeEventListener("abort", abortForShutdown);
+      releaseEligibility?.();
+      if (executionControllers.get(runId) === controller) executionControllers.delete(runId);
+    }
+  };
   const chains = new Map<string, Promise<void>>();
   const pending = new Set<string>();
+  // A runtime epoch gives recovery attempts fresh durable intent IDs after prior intents have
+  // already been delivered. Existing pending run.execute intents retain their persisted backoff.
+  const recoveryEpoch = randomUUID();
+  let recoverySequence = 0;
+  const recoveryReconciliationIntervalMs = 250;
+  let nextRecoveryReconciliationAt = 0;
+  const recoveryOutboxByRun = new Map<string, string>();
+  const deliveries = new Set<Promise<void>>();
+  const trackDelivery=(job:Promise<unknown>):void=>{
+    const tracked=job.then(()=>{},()=>{}).finally(()=>deliveries.delete(tracked));
+    deliveries.add(tracked);
+  };
   const execute = (runId: string): Promise<void> => {
     const next = (chains.get(runId) ?? Promise.resolve()).then(async () => {
-      for (let attempt = 0; attempt < 5 && !closed; attempt++) if ((await executeOnce(core, runId)) === "done") return;
+      const view=foldRun(runId,await store.readStream(runStream(runId)));
+      if(closed || !view || isTerminal(view))return;
+      const release=await acquireExecutionSlot(runId, view);
+      if (!release) return;
+      try{
+        if (closed) return;
+        for (let attempt = 0; attempt < 5 && !closed; attempt++) if ((await executeOnce(core, runId)) === "done") return;
+      }finally{release();}
     });
     const tracked = next.finally(() => {
       if (chains.get(runId) === tracked) chains.delete(runId);
@@ -74,7 +184,8 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     scopeId: kernel.root.id,
     recall: recallFor(store, options.cognition),
     outboxFor: options.outboxFor ?? (() => []),
-    controllers: new Map(),
+    controllers: executionControllers,
+    shutdownSignal: shutdown.signal,
     remoteOffers: new RemoteOfferRegistry(),
     execute,
     kick: () => kick(),
@@ -88,21 +199,74 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
 
   const runDispatcher = new OutboxDispatcher(store, {
     topic: RUN_EXECUTE_TOPIC,
+    concurrency: capacity,
+    admission: deliveryAdmission,
     backoff: DELIVERY,
     deliver: async (item) => {
-      await execute((item.payload as { runId: string }).runId);
-      return { executed: true };
+      const runId = (item.payload as { runId: string }).runId;
+      try {
+        await execute(runId);
+        return { executed: true };
+      } finally {
+        // The store retains pending retry/poison state; this map only correlates this process's
+        // currently delivered recovery intent and must not retain failed or exhausted attempts.
+        if (recoveryOutboxByRun.get(runId) === item.id) recoveryOutboxByRun.delete(runId);
+      }
     },
   });
 
+  async function persistRecoveryIntents(): Promise<void> {
+    // Reconcile only when a new recovery candidate exists. A tracked intent already has a
+    // durable retry schedule; rescanning the complete pending topic every timer tick is needless.
+    if (pending.size === 0) return;
+    const now = performance.now();
+    if (now < nextRecoveryReconciliationAt) return;
+    // Set the deadline before awaiting: a slow or failing store query must not hot-loop scans.
+    nextRecoveryReconciliationAt = now + recoveryReconciliationIntervalMs;
+    const pendingItems = await store.listOutbox({ topic: RUN_EXECUTE_TOPIC, status: "pending" });
+    const pendingItemIds = new Set(pendingItems.map((item) => item.id));
+    for (const [runId, id] of recoveryOutboxByRun) {
+      if (pendingItemIds.has(id)) pending.delete(runId);
+      else recoveryOutboxByRun.delete(runId);
+    }
+    const candidates = [...pending].filter((runId) => !recoveryOutboxByRun.has(runId)).slice(0, 100);
+    if (candidates.length === 0) return;
+    const pendingRunItems = new Map<string, string>();
+    for (const item of pendingItems) {
+      const runId = (item.payload as { runId?: unknown } | null)?.runId;
+      if (typeof runId === "string" && !pendingRunItems.has(runId)) pendingRunItems.set(runId, item.id);
+    }
+    const intents: { id: string; topic: string; payload: Json }[] = [];
+    const selected = new Map<string, string>();
+    for (const runId of candidates) {
+      const existing = pendingRunItems.get(runId);
+      const id = existing ?? `${RUN_EXECUTE_TOPIC}:recovery:${recoveryEpoch}:${++recoverySequence}:${runId}`;
+      selected.set(runId, id);
+      if (!existing) intents.push({ id, topic: RUN_EXECUTE_TOPIC, payload: { runId } });
+    }
+    if (intents.length > 0) await store.commit({ appends: [], outbox: intents });
+    for (const [runId, id] of selected) {
+      recoveryOutboxByRun.set(runId, id);
+      pending.delete(runId);
+    }
+  }
+
+  let resumeFirst = false;
   async function tick(): Promise<void> {
     for (const record of await continuations.expireDue()) pending.add(record.runId);
-    await continuations.dispatchResumes();
-    await runDispatcher.drain();
-    for (const runId of [...pending]) {
-      pending.delete(runId);
-      await execute(runId);
+    await persistRecoveryIntents();
+    // Dispatchers reserve a bounded set of slots across overlapping drains. A new runnable
+    // arrival can fill a free slot while an earlier long-running delivery is still executing.
+    // Rotate priority: even an empty topic briefly reserves capacity while its claim is
+    // in flight. A fixed first topic could otherwise starve the other at capacity one.
+    if (resumeFirst) {
+      trackDelivery(continuations.dispatchResumes());
+      trackDelivery(runDispatcher.drain());
+    } else {
+      trackDelivery(runDispatcher.drain());
+      trackDelivery(continuations.dispatchResumes());
     }
+    resumeFirst = !resumeFirst;
   }
 
   let ticking: Promise<void> | null = null;
@@ -148,7 +312,10 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     if (view.status === "running") pending.add(id);
     for (const continuationId of view.waitingOn) {
       const record = await continuations.get(continuationId);
-      if (record && !["OPEN", "BLOCKED", "RESUMED"].includes(record.status)) pending.add(id);
+      // A resume delivery can be acknowledged during shutdown before run.resumed is appended.
+      // Re-enter closed waits, including RESUMED, even when their outbox receipt is delivered.
+      // The run-local execution chain and recorded wait result make duplicate delivery harmless.
+      if (record && !["OPEN", "BLOCKED"].includes(record.status)) pending.add(id);
     }
   }
 
@@ -290,7 +457,7 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
   }
 
   async function quiescent(): Promise<boolean> {
-    if (ticking || pending.size > 0 || chains.size > 0) return false;
+    if (ticking || pending.size > 0 || chains.size > 0 || deliveries.size>0) return false;
     const now = Date.now();
     for (const topic of TOPICS) {
       const items = await store.listOutbox({ topic, status: "pending" });
@@ -339,9 +506,11 @@ export async function startRuntime(options: RuntimeOptions): Promise<Runtime> {
     async close() {
       if (closed) return;
       closed = true;
+      shutdown.abort();
       clearInterval(timer);
       if (approvalTimer) clearInterval(approvalTimer);
       await ticking;
+      await Promise.allSettled([...deliveries]);
       await Promise.allSettled([...chains.values()]);
       for (const runner of runners) await runner.stop();
       unregister();