@frockbot/kernel-do 0.3.4 → 0.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/kernel-do",
-  "version": "0.3.4",
+  "version": "0.3.6",
   "private": false,
   "type": "module",
   "exports": {
@@ -12,8 +12,8 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-composition": "0.3.4",
-    "@frockbot/kernel-contracts": "0.3.4",
+    "@frockbot/kernel-composition": "0.3.6",
+    "@frockbot/kernel-contracts": "0.3.6",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/authority.ts

@@ -34,11 +34,14 @@ import {
   eventsForFailedRun,
   latestModelRequestJournalState,
   planBotRunRecovery,
+  type ProviderReconcilesV1,
+  repairOrphanedOpenTurnV1,
   unresolvedModelRequestFailure,
 } from "./run-recovery.js";
 import {
   BotTurnReconciliationRequiredError,
   BotTurnRecoveryRequiredError,
+  BotTurnRefusedError,
 } from "./turn-errors.js";
 import {
   ACTIVE_RUN_KEY,
@@ -142,6 +145,16 @@ export interface BotDurableAuthorityHooks<Snapshot> {
     run: StoredRunV1<Snapshot>;
     read<T>(key: string): Promise<T | undefined>;
   }): Promise<Record<string, unknown>>;
+  /**
+   * Whether the named provider can be asked what happened to a model request
+   * it never answered (ADR 0028).
+   *
+   * Synchronous and pure, because it is consulted inside the recovery
+   * transaction: it answers from what the deployment knows about a provider
+   * Package, never by reaching one. Absent means every provider reconciles,
+   * which is the behaviour that predates the ADR.
+   */
+  providerReconciles?: ProviderReconcilesV1;
 }
 
 /** What a `turn/end` records when a later user message took a Turn's place. */
@@ -150,6 +163,22 @@ export const SUPERSEDED_TURN_REASON_V1 = "superseded by a new user message";
 /** How many times a queued Turn retries the object before giving up. */
 const MAX_QUEUED_RUN_START_ATTEMPTS = 8;
 
+/**
+ * True when this object has already durably decided to throw the Turn away.
+ *
+ * Reconciliation exists to retrieve an external outcome the Turn still needs.
+ * A Turn a Stop or a supersede has already discarded needs nothing: its
+ * provider outcome cannot change what it settles as, and parking it would keep
+ * the active-run marker — and so refuse every later message — over an answer
+ * nobody is waiting for. The intent the User expressed wins, and the run
+ * settles `cancelled` or `superseded` with everything it had already said.
+ */
+function runWasDiscardedV1(
+  run: { stopRequestedAt?: string; supersededAt?: string } | undefined,
+): boolean {
+  return Boolean(run?.stopRequestedAt || run?.supersededAt);
+}
+
 export interface BotDurableAuthorityOptions<Snapshot> {
   state: DurableObjectState;
   codec: StoredRunCodecV1<Snapshot>;
@@ -193,7 +222,14 @@ export class BotDurableAuthority<Snapshot> {
 
   async run(command: OwnedBotTurnCommand): Promise<BotTurnCompletion> {
     await this.assertMatchingRunCommand(command);
-    await this.recoverActiveRun();
+    // Recovering whatever this object was left holding must never decide the
+    // fate of a new command. `recoverActiveRun` executes the *previous* Turn
+    // inline and rethrows, so a recovery that failed — an uncertain effect, a
+    // mount failure, a provider that was down — threw before the new message
+    // was ever admitted, and the person's message was simply lost. The old
+    // Turn is durable either way and the alarm retries it; admission now
+    // refuses or supersedes on its own terms.
+    await this.recoverActiveRun().catch(() => undefined);
     const replay = await this.settledRunResult(command);
     if (replay) return replay;
     const admission = await this.acceptRun(command);
@@ -240,8 +276,10 @@ export class BotDurableAuthority<Snapshot> {
         const promoted = await this.promoteQueuedRun(command.runId);
         if (promoted === "blocked") {
           // Another Turn holds the object. Recovery drives it to its own
-          // durable terminal or resumable state, and this one tries again.
-          await this.recoverActiveRun();
+          // durable terminal or resumable state, and this one tries again —
+          // including when that recovery fails, which is the other Turn's
+          // problem and not this one's.
+          await this.recoverActiveRun().catch(() => undefined);
           // Unless what holds the object is an uncertain effect. That is
           // settled by an explicit reconciliation the User asks for, on their
           // own clock, and retrying against it would only burn this caller's
@@ -249,11 +287,14 @@ export class BotDurableAuthority<Snapshot> {
           // run is durable: it stays queued, and the reconciliation's own
           // settlement — or the recovery alarm — starts it.
           if (await this.activeRunAwaitsReconciliation()) {
-            return {
-              runId: command.runId,
-              text: "",
-              events: [],
-            } satisfies BotTurnCompletion;
+            // A Turn that has not run is not a completed Turn. Answering with
+            // an empty completion made the browser render the person's new
+            // message as answered with silence; the durable queue entry stays,
+            // and the refusal says why nothing has happened yet.
+            throw new BotTurnRefusedError(
+              "reconciliation-required",
+              `run "${command.runId}" is queued: the active run requires reconciliation before another Turn can be admitted`,
+            );
           }
           continue;
         }
@@ -369,12 +410,22 @@ export class BotDurableAuthority<Snapshot> {
         (await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
       ).map(decodeSessionEvent);
       const settings = run.configurationSnapshot;
-      await transaction.put(key, {
-        ...run,
-        status: "running",
-        phase: "executing",
-        failure: undefined,
-      } satisfies StoredRunV1<Snapshot>);
+      // The failure is *removed*, not set to `undefined`: a running run that
+      // carries a `failure` key is a shape the run record does not allow, and
+      // writing one turned "resolve this Turn" into a record nothing could
+      // read afterwards. `require` checks it here, where the write is, rather
+      // than leaving the projector to fail on every later read.
+      const { failure: _failure, ...resumed } = run;
+      await transaction.put(
+        key,
+        structuredClone(
+          this.codec.require({
+            ...resumed,
+            status: "running",
+            phase: "executing",
+          } satisfies StoredRunV1<Snapshot>),
+        ),
+      );
       await this.refreshRecoveryAlarm(transaction);
       return { run, latest, settings };
     });
@@ -480,8 +531,9 @@ export class BotDurableAuthority<Snapshot> {
       }
       const modelState = latestModelRequestJournalState(events);
       if (
-        error instanceof BotTurnReconciliationRequiredError ||
-        modelState.status === "unresolved"
+        (error instanceof BotTurnReconciliationRequiredError ||
+          modelState.status === "unresolved") &&
+        !runWasDiscardedV1(durableRun)
       ) {
         await this.requireRunReconciliation(
           command.runId,
@@ -612,8 +664,9 @@ export class BotDurableAuthority<Snapshot> {
       }
       const modelState = latestModelRequestJournalState(events);
       if (
-        error instanceof BotTurnReconciliationRequiredError ||
-        modelState.status === "unresolved"
+        (error instanceof BotTurnReconciliationRequiredError ||
+          modelState.status === "unresolved") &&
+        !runWasDiscardedV1(durableRun)
       ) {
         await this.requireRunReconciliation(
           run.runId,
@@ -659,7 +712,8 @@ export class BotDurableAuthority<Snapshot> {
     );
     if (!run) return undefined;
     if (run.commandFingerprint !== botTurnCommandFingerprintV1(command)) {
-      throw new Error(
+      throw new BotTurnRefusedError(
+        "duplicate",
         `Turn idempotency key "${runId}" was reused for a different command`,
       );
     }
@@ -674,7 +728,8 @@ export class BotDurableAuthority<Snapshot> {
       };
     }
     if (run.status !== "completed") {
-      throw new Error(
+      throw new BotTurnRefusedError(
+        "duplicate",
         `run "${runId}" already exists with status ${run.status}`,
       );
     }
@@ -702,7 +757,8 @@ export class BotDurableAuthority<Snapshot> {
       run &&
       run.commandFingerprint !== botTurnCommandFingerprintV1(command)
     ) {
-      throw new Error(
+      throw new BotTurnRefusedError(
+        "duplicate",
         `Turn idempotency key "${command.runId}" was reused for a different command`,
       );
     }
@@ -788,7 +844,16 @@ export class BotDurableAuthority<Snapshot> {
         this.ctx.storage.get<BotIdentity>(IDENTITY_KEY),
       ]);
       const run = this.codec.optional(storedRun);
-      if (run?.status === "reconciliation-required" && identity) return;
+      if (run?.status === "reconciliation-required" && identity) {
+        // A parked run is not this alarm's to settle — only an explicit
+        // reconciliation settles it — but returning without rescheduling
+        // dropped the object's *other* deadlines with it: a Routine due while
+        // a Bot sat parked never fired, and nothing set the alarm again.
+        await this.ctx.storage.transaction((transaction) =>
+          this.refreshRecoveryAlarm(transaction),
+        );
+        return;
+      }
     }
     await this.recoverActiveRun();
   }
@@ -855,17 +920,15 @@ export class BotDurableAuthority<Snapshot> {
         const storedFences = storedRunAdmissionFences(
           await transaction.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
         );
-        if (
-          !storedFences.includes(runId) &&
-          storedFences.length >= MAX_RUN_ADMISSION_FENCES
-        ) {
-          throw new Error("Run admission fence capacity reached");
-        }
+        // A bounded FIFO, not a cliff. Nothing ever evicted an entry, so a Bot
+        // that had refused 256 sends over its life answered every later fence
+        // with a 500 and left the client retrying "Turn admission lookup
+        // failed" forever. A run id old enough to age out here can no longer
+        // be admitted by any live caller.
+        const kept = storedFences.filter((fenced) => fenced !== runId);
+        while (kept.length >= MAX_RUN_ADMISSION_FENCES) kept.shift();
         await transaction.put({
-          [RUN_ADMISSION_FENCE_INDEX_KEY]: [
-            ...storedFences.filter((fenced) => fenced !== runId),
-            runId,
-          ],
+          [RUN_ADMISSION_FENCE_INDEX_KEY]: [...kept, runId],
           [IDENTITY_KEY]: durableIdentity ?? identity,
         });
         await transaction.delete(`${RUN_ADMISSION_FENCE_PREFIX}${runId}`);
@@ -932,7 +995,10 @@ export class BotDurableAuthority<Snapshot> {
       fences.includes(command.runId) ||
       (await this.ctx.storage.get(fenceKey))
     ) {
-      throw new Error(`run "${command.runId}" admission was fenced`);
+      throw new BotTurnRefusedError(
+        "fenced",
+        `run "${command.runId}" admission was fenced`,
+      );
     }
     const settings = await this.hooks.resolveAdmissionSnapshot(command);
     // Materialized before the transaction; the pin itself is read inside it.
@@ -944,20 +1010,30 @@ export class BotDurableAuthority<Snapshot> {
         if (
           existing.commandFingerprint !== botTurnCommandFingerprintV1(command)
         ) {
-          throw new Error(
+          throw new BotTurnRefusedError(
+            "duplicate",
             `Turn idempotency key "${command.runId}" was reused for a different command`,
           );
         }
         if (existing.status === "completed") {
-          throw new Error(`run "${command.runId}" already completed`);
+          throw new BotTurnRefusedError(
+            "duplicate",
+            `run "${command.runId}" already completed`,
+          );
         }
-        throw new Error(`run "${command.runId}" already exists`);
+        throw new BotTurnRefusedError(
+          "duplicate",
+          `run "${command.runId}" already exists`,
+        );
       }
       const fences = storedRunAdmissionFences(
         await transaction.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
       );
       if (fences.includes(command.runId) || (await transaction.get(fenceKey))) {
-        throw new Error(`run "${command.runId}" admission was fenced`);
+        throw new BotTurnRefusedError(
+          "fenced",
+          `run "${command.runId}" admission was fenced`,
+        );
       }
       const identity = await transaction.get<BotIdentity>(IDENTITY_KEY);
       if (
@@ -970,9 +1046,42 @@ export class BotDurableAuthority<Snapshot> {
       const supersede = activeRunId
         ? await this.planSupersede(transaction, command, activeRunId)
         : undefined;
-      const latestEvents = (
+      const storedEvents = (
         (await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
       ).map(decodeSessionEvent);
+      // A Turn that died between `turn/start` and `turn/end` — an event the
+      // encoder refused, a durable write that failed — left the log open, and
+      // every later Turn failed validation with "turn N started while turn
+      // N-1 is open". Nothing owned that repair, because the run that would
+      // have closed it is already terminal, so admission does: with nothing
+      // executing, an open Turn is one nobody is going to finish.
+      //
+      // The pointer alone is not the test. A Bot can hold an `active-run` id
+      // whose record is already terminal — a settlement that landed while the
+      // pointer clear did not, a supersede whose Turn ended between the two
+      // writes — and gating the repair on the pointer left exactly those Bots
+      // wedged. What matters is whether anything is still entitled to write
+      // that Turn's end: a `running` record is, and so is a
+      // `reconciliation-required` one, whose Turn is held open on purpose
+      // until its outcome is retrieved. Nothing else is.
+      const activeRun = activeRunId
+        ? this.codec.optional(
+            await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
+          )
+        : undefined;
+      const stillOwned =
+        activeRun?.status === "running" ||
+        activeRun?.status === "reconciliation-required";
+      const repairs = stillOwned
+        ? []
+        : repairOrphanedOpenTurnV1(command.sessionId, storedEvents);
+      const latestEvents = [...storedEvents, ...repairs];
+      if (repairs.length > 0) {
+        await transaction.put(
+          LATEST_EVENTS_KEY,
+          structuredClone(latestEvents.map(decodeSessionEvent)),
+        );
+      }
       const admittedSettings = await this.hooks.admittedSnapshot(
         transaction,
         settings,
@@ -1060,21 +1169,23 @@ export class BotDurableAuthority<Snapshot> {
     // no run when the person pressed send — supersedes exactly as a named one
     // does; only an absent field is "no intent", and that is still refused.
     if (lane !== "user" || !command.supersedes) {
-      throw new Error("bot already has an active run");
+      throw new BotTurnRefusedError("busy", "bot already has an active run");
     }
     const active = this.codec.optional(
       await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
     );
-    if (!active) throw new Error("bot already has an active run");
+    if (!active)
+      throw new BotTurnRefusedError("busy", "bot already has an active run");
     if (active.status === "reconciliation-required") {
       // An uncertain external effect is never abandoned to admit something
       // else: the outcome has to be retrieved before this object runs again.
-      throw new Error(
+      throw new BotTurnRefusedError(
+        "reconciliation-required",
         `run "${activeRunId}" requires reconciliation before another Turn can be admitted`,
       );
     }
     if (active.status !== "running") {
-      throw new Error("bot already has an active run");
+      throw new BotTurnRefusedError("busy", "bot already has an active run");
     }
     const pendingRunId = await transaction.get<string>(PENDING_RUN_KEY);
     // A Turn that has not dispatched a model request has no durable work to
@@ -1335,7 +1446,12 @@ export class BotDurableAuthority<Snapshot> {
       const latest = (
         (await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
       ).map(decodeSessionEvent);
-      const plan = planBotRunRecovery(run, latest, this.codec);
+      const plan = planBotRunRecovery(
+        run,
+        latest,
+        this.codec,
+        this.hooks.providerReconciles ?? (() => true),
+      );
       if (plan.kind === "complete") {
         const result = {
           runId: run.runId,
@@ -1360,13 +1476,19 @@ export class BotDurableAuthority<Snapshot> {
         return undefined;
       }
       if (plan.kind === "fail") {
+        // The repairs matter when the failure is ADR 0028's: they close the
+        // tool occurrences the restart left open, so the settled run's journal
+        // is a complete account rather than one that stops mid-sentence twice.
+        const events = plan.repairs
+          ? [...run.events, ...plan.repairs]
+          : run.events;
         await failStoredRun(
           this.codec,
           transaction,
           this.terminalKeys(run.runId),
           run.runId,
           latest.slice(0, run.previousEventCount),
-          run.events,
+          events,
           plan.failure,
           this.supersededPackageRecords(),
         );
@@ -1400,6 +1522,22 @@ export class BotDurableAuthority<Snapshot> {
         await this.refreshRecoveryAlarm(transaction);
         return { kind: "resume" as const, run, latest, settings };
       }
+      if (runWasDiscardedV1(run)) {
+        // Recovery of a Turn Stop or supersede already discarded settles it on
+        // that intent rather than parking it: nothing is owed the answer.
+        await failStoredRun(
+          this.codec,
+          transaction,
+          this.terminalKeys(run.runId),
+          run.runId,
+          latest.slice(0, run.previousEventCount),
+          [...run.events, ...plan.repairs],
+          "Execution outcome requires reconciliation before it can resume",
+          this.supersededPackageRecords(),
+        );
+        await this.refreshRecoveryAlarm(transaction);
+        return undefined;
+      }
       await transaction.put({
         [key]: {
           ...run,

package/src/run-records.ts

@@ -758,7 +758,15 @@ export function botTurnCommandFingerprintV1(
       ...(lane === defaultRunLaneV1(turnType) ? {} : { lane }),
       ...(command.subagentRole ? { subagentRole: command.subagentRole } : {}),
       ...(command.origin ? { origin: command.origin } : {}),
-      ...(command.supersedes ? { supersedes: command.supersedes } : {}),
+      // The *intent* is part of the command's identity, exactly as ADR 0024
+      // requires: a replay of a command that carried no supersede can never
+      // become one that interrupts a second Turn. The provenance is not. A
+      // client retrying the same send — same commandId, same text — names
+      // whichever run it happened to have observed by then, and that is a fact
+      // about its polling, not about what the person asked for. Hashing it
+      // turned an ordinary retry into "this idempotency key was reused for a
+      // different command" and refused the send.
+      ...(command.supersedes ? { supersedes: true } : {}),
       ...(skills.length > 0 ? { skills: skills.map(formatSkillRefV1) } : {}),
       ...(command.directTool ? { directTool: command.directTool } : {}),
     })}`;

package/src/run-recovery.test.ts

@@ -2,8 +2,11 @@ import { describe, expect, test } from "bun:test";
 import type { SessionEvent } from "@frockbot/kernel-contracts";
 import {
   latestModelRequestJournalState,
+  planBotRunRecovery,
+  UNRECONCILABLE_RUN_FAILURE_V1,
   unresolvedModelRequestFailure,
 } from "./run-recovery.js";
+import { createStoredRunCodecV1, type StoredRunV1 } from "./run-records.js";
 
 // Distributive, so each member of the union keeps its own fields: a bare
 // `Omit` over the union collapses to the keys they all share.
@@ -76,18 +79,6 @@ describe("unresolvedModelRequestFailure", () => {
     );
   });
 
-  test("reads the last reason when the Turn was retried", () => {
-    expect(
-      unresolved(
-        request,
-        reconciliationRequired("request-1", "first attempt"),
-        reconciliationRequired("request-1", "second attempt"),
-      ),
-    ).toBe(
-      'Model request "request-1" has no durable provider outcome: second attempt',
-    );
-  });
-
   test("ignores a reason journaled against another request", () => {
     expect(
       unresolved(
@@ -104,3 +95,120 @@ describe("unresolvedModelRequestFailure", () => {
     );
   });
 });
+
+// ADR 0028. A restart mid-Turn used to park every in-flight run on a
+// reconciliation nobody could perform: the providers this deployment actually
+// uses expose no response retrieval, so the banner's Resolve action had one
+// possible outcome and the Bot stayed wedged until somebody clicked it.
+describe("a restart with no retrievable provider outcome", () => {
+  const codec = createStoredRunCodecV1<null>({
+    decodeRunId: (value) => String(value),
+    decodeConfigurationSnapshot: () => null,
+  });
+
+  function runWith(events: SessionEvent[]): StoredRunV1<null> {
+    return {
+      runId: "run-1",
+      commandFingerprint: "fingerprint-1",
+      sessionId: "user-1:bot-1",
+      acceptedAt: new Date(Date.UTC(2026, 8, 3)).toISOString(),
+      input: "hello",
+      events,
+      effectAdmissions: [],
+      status: "running",
+      phase: "executing",
+      compositionGenerationId: "generation-1",
+      configurationSnapshot: null,
+      previousEventCount: 0,
+    };
+  }
+
+  /** `Session` requires a zero-based contiguous log; this file's own stamper is one-based. */
+  function durableJournal(...events: UnstampedEvent[]): SessionEvent[] {
+    return events.map(
+      (event, index) =>
+        ({
+          ...event,
+          seq: index,
+          timestamp: new Date(Date.UTC(2026, 8, 3, 0, 0, index)).toISOString(),
+        }) as SessionEvent,
+    );
+  }
+
+  const openTurn: UnstampedEvent[] = [
+    {
+      type: "session/created",
+      createdAt: new Date(Date.UTC(2026, 8, 3)).toISOString(),
+    },
+    { type: "input/queued", messageId: "message-1", text: "hello" },
+    { type: "turn/start", turn: 1 },
+    { type: "step/start", turn: 1, step: 1 },
+    {
+      type: "user/message",
+      turn: 1,
+      step: 1,
+      messageId: "message-1",
+      text: "hello",
+    },
+  ];
+
+  test("settles the run as failed rather than parking it", () => {
+    const events = durableJournal(...openTurn, request);
+    const plan = planBotRunRecovery(
+      runWith(events),
+      events,
+      codec,
+      (provider) => provider === "foundation",
+    );
+
+    expect(plan.kind).toBe("fail");
+    expect(plan.kind === "fail" ? plan.failure : "").toBe(
+      UNRECONCILABLE_RUN_FAILURE_V1,
+    );
+    // Whatever repairs the resume would have written travel with the
+    // settlement, so an unresolved tool occurrence is closed rather than left
+    // open in a record nothing will ever revisit. This journal needs none.
+    expect(plan.kind === "fail" ? plan.repairs : undefined).toEqual([]);
+  });
+
+  test("keeps parking a run whose provider can be asked", () => {
+    const events = durableJournal(...openTurn, request);
+    const plan = planBotRunRecovery(runWith(events), events, codec, () => true);
+
+    expect(plan.kind).toBe("reconcile");
+  });
+
+  test("parks by default, so a host that names no policy is unaffected", () => {
+    const events = durableJournal(...openTurn, request);
+
+    expect(planBotRunRecovery(runWith(events), events, codec).kind).toBe(
+      "reconcile",
+    );
+  });
+
+  test("preserves the words the Turn had already streamed", () => {
+    const events = durableJournal(...openTurn, request, {
+      type: "assistant/chunk",
+      turn: 1,
+      step: 1,
+      requestId: "request-1",
+      text: "Half a thought",
+    });
+    const plan = planBotRunRecovery(
+      runWith(events),
+      events,
+      codec,
+      () => false,
+    );
+
+    expect(plan.kind).toBe("fail");
+    // Nothing in the plan discards the journal: the settled record carries the
+    // run's own events, and the projection reads the partial answer back out.
+    expect(
+      events.some(
+        (event) =>
+          event.type === "assistant/chunk" && event.text === "Half a thought",
+      ),
+    ).toBe(true);
+  });
+});