@frockbot/kernel-do 0.3.4 → 0.3.6

package/src/run-recovery.ts

@@ -11,11 +11,42 @@ import type { StoredRunCodecV1, StoredRunV1 } from "./run-records.js";
 
 export type BotRunRecoveryPlan =
   | { kind: "complete"; responseText: string }
-  | { kind: "fail"; failure: string }
+  | { kind: "fail"; failure: string; repairs?: SessionEvent[] }
   | { kind: "restart"; previous: SessionEvent[] }
   | { kind: "resume" }
   | { kind: "reconcile"; repairs: SessionEvent[] };
 
+/**
+ * What a Turn says when a restart caught it mid-answer and nobody can be asked
+ * how it ended (ADR 0028).
+ *
+ * It is written for the person watching, not for an operator: they saw the Bot
+ * start talking and then stop, and the only useful thing to tell them is that
+ * it will not be finishing that sentence and sending again is safe.
+ */
+export const UNRECONCILABLE_RUN_FAILURE_V1 =
+  "This Turn stopped partway — the service restarted while the model was answering, and there is no way to find out how that request ended. Try sending it again.";
+
+/**
+ * Whether the provider a run was talking to can be asked what happened to a
+ * request it never answered.
+ *
+ * Given the provider id off the run's own durable `model/request`, so the
+ * answer is the same on every recovery of the same run, with no dependency on
+ * what happens to be mounted or resident.
+ */
+export type ProviderReconcilesV1 = (providerId: string) => boolean;
+
+/** The provider the run's most recent durable model request was addressed to. */
+export function latestModelRequestProviderV1(
+  events: readonly SessionEvent[],
+): string | undefined {
+  const request = events.findLast((event) => event.type === "model/request");
+  return request?.type === "model/request"
+    ? request.request.provider
+    : undefined;
+}
+
 export type ModelRequestJournalState =
   | { status: "none" }
   | {
@@ -90,6 +121,7 @@ export function planBotRunRecovery<Snapshot>(
   run: StoredRunV1<Snapshot>,
   latest: readonly SessionEvent[],
   codec: StoredRunCodecV1<Snapshot>,
+  providerReconciles: ProviderReconcilesV1 = () => true,
 ): BotRunRecoveryPlan {
   codec.require(run);
   let toolJournal: ReturnType<typeof validateToolOccurrenceJournal>;
@@ -179,7 +211,59 @@ export function planBotRunRecovery<Snapshot>(
     };
   }
   const session = new Session(run.sessionId, () => {}, latest);
-  return { kind: "reconcile", repairs: session.reconcileForResume() };
+  const repairs = session.reconcileForResume();
+  // ADR 0028. A Turn whose model outcome is unknown is parked only when
+  // somebody can actually be asked. When the provider offers no retrieval,
+  // parking is not caution — it is a dead end: nothing will ever arrive to
+  // resolve it, the Bot stays wedged behind it, and the person is handed a
+  // Resolve button whose only possible answer is "give up". So the run is
+  // settled `failed` here, with its repairs and every streamed word it had
+  // already sent kept in the journal.
+  const provider = latestModelRequestProviderV1(run.events);
+  if (provider !== undefined && !providerReconciles(provider)) {
+    return { kind: "fail", failure: UNRECONCILABLE_RUN_FAILURE_V1, repairs };
+  }
+  return { kind: "reconcile", repairs };
+}
+
+/** True when the durable log ends inside a Turn nothing is going to finish. */
+export function hasOrphanedOpenTurnV1(
+  events: readonly SessionEvent[],
+): boolean {
+  let openTurn: number | undefined;
+  for (const event of events) {
+    if (event.type === "turn/start") openTurn = event.turn;
+    if (event.type === "turn/end" && event.turn === openTurn) {
+      openTurn = undefined;
+    }
+  }
+  return openTurn !== undefined;
+}
+
+/**
+ * Closes a Turn the log was left inside, so the next one can start.
+ *
+ * A Turn that threw between `turn/start` and `turn/end` — an event the
+ * encoder refused, a durable write that failed — leaves an open turn in the
+ * durable log, and the next Turn on that Bot fails validation with "turn N
+ * started while turn N-1 is open". Forever: nothing owned the repair, because
+ * the run that would have written the `turn/end` is already terminal. This is
+ * that repair, applied when no run is executing, so an interrupted Turn is
+ * recorded as interrupted rather than wedging the Bot.
+ *
+ * A log too malformed to reconcile is left exactly as it is: repairing it
+ * blindly would invent history.
+ */
+export function repairOrphanedOpenTurnV1(
+  sessionId: string,
+  latest: readonly SessionEvent[],
+): SessionEvent[] {
+  if (!hasOrphanedOpenTurnV1(latest)) return [];
+  try {
+    return new Session(sessionId, () => {}, latest).reconcileInterrupted();
+  } catch {
+    return [];
+  }
 }
 
 export function eventsForFailedRun(

package/src/run-terminal-open-turn.test.ts

@@ -0,0 +1,203 @@
+// The exact sequence that wedged a Bot forever.
+//
+// A Turn interrupted mid-answer unwinds without writing a `turn/end`: its model
+// request has no durable outcome and a `turn/end` would claim to know how it
+// ended. Right while the run might resume; wrong once it will not. The
+// settlement committed those events as they stood, so the durable session log
+// ended inside an open turn, and every later message on that Bot failed
+// validation with `turn 2 started while turn 1 is open` — printed verbatim into
+// the person's next bubble, forever.
+import { describe, expect, test } from "bun:test";
+import {
+  Session,
+  type SessionEvent,
+  validateToolOccurrenceJournal,
+} from "@frockbot/kernel-contracts";
+import { MemoryStorage } from "./memory-storage.fixture.ts";
+import { createStoredRunCodecV1, type StoredRunV1 } from "./run-records.ts";
+import {
+  cancelStoredRun,
+  failStoredRun,
+  supersedeStoredRun,
+} from "./run-terminal.ts";
+
+const codec = createStoredRunCodecV1<null>({
+  decodeRunId: (value) => String(value),
+  decodeConfigurationSnapshot: () => null,
+});
+
+const SESSION_ID = "user-1:primary";
+
+const KEYS = {
+  run: "run:run-1",
+  activeRun: "active-run",
+  latestEvents: "latest-events",
+  notificationPrefix: "notification:",
+};
+
+/**
+ * A Turn stopped after its model request went uncertain: `turn/start`,
+ * `step/start`, a `user/message`, a `model/request`, and a
+ * `model/reconciliation-required` — and then nothing. This is what the Agent
+ * loop leaves behind when it unwinds on an abort.
+ */
+function interruptedJournal(): SessionEvent[] {
+  const events: SessionEvent[] = [];
+  const session = new Session(SESSION_ID, (envelope) => {
+    events.push(envelope.event);
+  });
+  session.appendBatch([
+    { type: "turn/start", turn: 1 },
+    { type: "step/start", turn: 1, step: 1 },
+    {
+      type: "user/message",
+      turn: 1,
+      step: 1,
+      messageId: "message-1",
+      text: "hello",
+    },
+    {
+      type: "model/request",
+      turn: 1,
+      step: 1,
+      request: {
+        requestId: "request-1",
+        provider: "flock-ai",
+        model: "@flock/auto",
+        system: "",
+        messages: [],
+        tools: [],
+      },
+    },
+    {
+      type: "model/reconciliation-required",
+      turn: 1,
+      step: 1,
+      requestId: "request-1",
+      reason: "Model response outcome is uncertain after cancellation",
+    },
+  ]);
+  return events;
+}
+
+function storedRun(
+  events: SessionEvent[],
+  intent: Partial<StoredRunV1<null>>,
+): StoredRunV1<null> {
+  return {
+    runId: "run-1",
+    commandFingerprint: "fingerprint-1",
+    sessionId: SESSION_ID,
+    acceptedAt: "2026-09-03T00:00:00.000Z",
+    input: "hello",
+    events,
+    effectAdmissions: [],
+    status: "running",
+    phase: "executing",
+    compositionGenerationId: "generation-1",
+    configurationSnapshot: null,
+    previousEventCount: 0,
+    ...intent,
+  };
+}
+
+async function settled(
+  intent: Partial<StoredRunV1<null>>,
+  settle: (storage: MemoryStorage, events: SessionEvent[]) => Promise<unknown>,
+): Promise<{ storage: MemoryStorage; latest: SessionEvent[] }> {
+  const storage = new MemoryStorage();
+  const events = interruptedJournal();
+  await storage.put({
+    [KEYS.activeRun]: "run-1",
+    [KEYS.run]: storedRun(events, intent),
+    [KEYS.latestEvents]: events,
+  });
+
+  await settle(storage, events);
+
+  return {
+    storage,
+    latest: storage.values.get(KEYS.latestEvents) as SessionEvent[],
+  };
+}
+
+/** What the next Turn does: start turn 2 on the log the settlement left. */
+function admitNextTurn(latest: SessionEvent[]): void {
+  const session = new Session(SESSION_ID, () => {}, latest);
+  session.append({ type: "turn/start", turn: 2 });
+  validateToolOccurrenceJournal(session.events);
+}
+
+describe("settling a Turn interrupted mid-answer", () => {
+  test("a superseded run leaves a log the next Turn can start on", async () => {
+    const { latest, storage } = await settled(
+      { supersededAt: "2026-09-03T00:01:00.000Z", supersededBy: "run-2" },
+      (store, events) =>
+        supersedeStoredRun(codec, store, KEYS, "run-1", [], events),
+    );
+
+    // Before the fix this threw "turn 2 started while turn 1 is open", and
+    // every later message on this Bot answered 500 with that sentence.
+    expect(() => admitNextTurn(latest)).not.toThrow();
+    expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
+    // The settled record carries the same closed account, not a different one.
+    const record = storage.values.get(KEYS.run) as StoredRunV1<null>;
+    expect(record.status).toBe("superseded");
+    expect(record.events.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
+  });
+
+  test("a stopped run leaves a log the next Turn can start on", async () => {
+    const { latest } = await settled(
+      { stopRequestedAt: "2026-09-03T00:01:00.000Z" },
+      (store, events) =>
+        cancelStoredRun(codec, store, KEYS, "run-1", [], events),
+    );
+
+    expect(() => admitNextTurn(latest)).not.toThrow();
+    expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
+  });
+
+  test("a failed run leaves a log the next Turn can start on", async () => {
+    const { latest } = await settled({}, (store, events) =>
+      failStoredRun(
+        codec,
+        store,
+        KEYS,
+        "run-1",
+        [],
+        events,
+        "the service restarted",
+      ),
+    );
+
+    expect(() => admitNextTurn(latest)).not.toThrow();
+    expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
+  });
+
+  test("a Turn that closed itself is not closed twice", async () => {
+    const storage = new MemoryStorage();
+    const events: SessionEvent[] = [];
+    const session = new Session(SESSION_ID, (envelope) => {
+      events.push(envelope.event);
+    });
+    session.appendBatch([
+      { type: "turn/start", turn: 1 },
+      { type: "step/start", turn: 1, step: 1 },
+      { type: "step/end", turn: 1, step: 1, outcome: "cancelled" },
+      { type: "turn/end", turn: 1, outcome: "cancelled" },
+    ]);
+    await storage.put({
+      [KEYS.activeRun]: "run-1",
+      [KEYS.run]: storedRun(events, {
+        stopRequestedAt: "2026-09-03T00:01:00.000Z",
+      }),
+      [KEYS.latestEvents]: events,
+    });
+
+    await cancelStoredRun(codec, storage, KEYS, "run-1", [], events);
+
+    const latest = storage.values.get(KEYS.latestEvents) as SessionEvent[];
+    expect(latest.filter((event) => event.type === "turn/end")).toHaveLength(1);
+    expect(() => admitNextTurn(latest)).not.toThrow();
+  });
+});

package/src/run-terminal.ts

@@ -1,5 +1,6 @@
 import {
   decodeSessionEvent,
+  Session,
   type SessionEvent,
 } from "@frockbot/kernel-contracts";
 import type {
@@ -8,6 +9,47 @@ import type {
   StoredRunV1,
 } from "./run-records.js";
 
+/**
+ * The events a terminal settlement commits, with any Turn they were left
+ * inside closed.
+ *
+ * A Turn interrupted mid-answer unwinds without writing a `turn/end`: the
+ * outcome of its model request is unknown, and a `turn/end` would claim to know
+ * how it ended. That is right while the run might still resume — and wrong the
+ * moment it will not. Settling one and committing its events as they stand left
+ * an open turn in the durable session log, so `turn N started while turn N-1 is
+ * open` refused every later message on that Bot, forever, and printed itself
+ * verbatim into the person's next bubble.
+ *
+ * So closing the turn happens exactly here: at the one point where the run is
+ * certainly not resuming. `reconcileInterrupted` writes the same repair the
+ * recovery path already writes — every unresolved tool occurrence closed as
+ * `interrupted`, then `step/end` and `turn/end` — so the settled log is a
+ * complete account and the next Turn starts on a closed one.
+ *
+ * A log that is already closed produces no repairs, and one too malformed to
+ * reconcile is left exactly as it is: repairing that blindly would invent
+ * history.
+ */
+function settledEventsV1(
+  sessionId: string,
+  previous: readonly SessionEvent[],
+  events: readonly SessionEvent[],
+): { events: SessionEvent[]; latestEvents: SessionEvent[] } {
+  const decoded = events.map(decodeSessionEvent);
+  const latest = [...previous, ...decoded].map(decodeSessionEvent);
+  let repairs: SessionEvent[] = [];
+  try {
+    repairs = new Session(sessionId, () => {}, latest).reconcileInterrupted();
+  } catch {
+    repairs = [];
+  }
+  return {
+    events: [...decoded, ...repairs],
+    latestEvents: [...latest, ...repairs],
+  };
+}
+
 export interface RunTerminalStorage {
   get<T>(key: string): Promise<T | undefined>;
   put(entries: Record<string, unknown>): Promise<void>;
@@ -83,7 +125,8 @@ export async function supersedeStoredRun<Snapshot>(
   if (!run.supersededAt) {
     throw new Error(`run "${runId}" has no durable supersede intent`);
   }
-  const decodedEvents = events.map(decodeSessionEvent);
+  const settledEvents = settledEventsV1(run.sessionId, previous, events);
+  const decodedEvents = settledEvents.events;
   const { responseText: _text, failure: _failure, ...settled } = run;
   // A run superseded while still queued never started, never appended an
   // event, and never spoke: it settles as a record on its own and leaves both
@@ -105,9 +148,7 @@ export async function supersedeStoredRun<Snapshot>(
     ...(queued
       ? {}
       : {
-          [keys.latestEvents]: structuredClone(
-            [...previous, ...decodedEvents].map(decodeSessionEvent),
-          ),
+          [keys.latestEvents]: structuredClone(settledEvents.latestEvents),
         }),
   };
   if (packageRecords && !queued) {
@@ -226,8 +267,9 @@ export async function cancelStoredRun<Snapshot>(
   if (!run.stopRequestedAt) {
     throw new Error(`run "${runId}" has no durable stop intent`);
   }
-  const decodedEvents = events.map(decodeSessionEvent);
-  const latestEvents = [...previous, ...decodedEvents].map(decodeSessionEvent);
+  const settledEvents = settledEventsV1(run.sessionId, previous, events);
+  const decodedEvents = settledEvents.events;
+  const latestEvents = settledEvents.latestEvents;
   const { responseText: _text, failure: _failure, ...settled } = run;
   const cancelled = codec.require({
     ...settled,
@@ -278,8 +320,9 @@ export async function failStoredRun<Snapshot>(
       supersededRecords,
     );
   }
-  const decodedEvents = events.map(decodeSessionEvent);
-  const latestEvents = [...previous, ...decodedEvents].map(decodeSessionEvent);
+  const settledEvents = settledEventsV1(run.sessionId, previous, events);
+  const decodedEvents = settledEvents.events;
+  const latestEvents = settledEvents.latestEvents;
   const failed = codec.require({
     ...run,
     events: decodedEvents,

package/src/turn-admission.test.ts

@@ -448,3 +448,41 @@ describe("an admitted Turn re-mounts on its recorded turn type", () => {
     });
   });
 });
+
+// robustness F18. The composer sends `supersedes` on every send and names
+// whichever run it happened to have observed. A retry of the same send names a
+// different one — or none — and used to be refused as a reused idempotency key.
+describe("a retried send is idempotent whatever run it names", () => {
+  const command = {
+    userId: "user-1",
+    botId: "primary",
+    runId: "run-1",
+    sessionId: "user-1:primary",
+    acceptedAt: "2026-08-31T01:00:00.000Z",
+    text: "hello",
+    lane: "user" as const,
+  };
+
+  test("the observed run id is not part of the command's identity", () => {
+    const first = botTurnCommandFingerprintV1({ ...command, supersedes: {} });
+
+    expect(
+      botTurnCommandFingerprintV1({
+        ...command,
+        supersedes: { runId: "run-0" },
+      }),
+    ).toBe(first);
+    expect(
+      botTurnCommandFingerprintV1({
+        ...command,
+        supersedes: { runId: "run-99" },
+      }),
+    ).toBe(first);
+  });
+
+  test("but the intent itself still is, so a replay cannot gain one", () => {
+    expect(
+      botTurnCommandFingerprintV1({ ...command, supersedes: {} }),
+    ).not.toBe(botTurnCommandFingerprintV1(command));
+  });
+});

package/src/turn-errors.ts

@@ -25,6 +25,50 @@ export class BotTurnReconciliationRequiredError extends Error {
   }
 }
 
+/** Why the Bot declined to admit a Turn. */
+export type BotTurnRefusalCodeV1 =
+  "busy" | "reconciliation-required" | "fenced" | "duplicate";
+
+const BOT_TURN_REFUSAL_PREFIX_V1 = "BotTurnRefusedError:";
+
+/**
+ * An admission the Bot declined, as a typed error rather than a sentence.
+ *
+ * A refusal crosses a Durable Object RPC boundary, which preserves an error's
+ * `name` and `message` and drops everything else — so the code rides on the
+ * name, the way the gateway already relies on `name` for `BotNotFoundError`.
+ * Classifying these by matching prose against `error.message` meant any
+ * reword silently turned an ordinary 409 into a 500, and two real messages
+ * already fell through.
+ */
+export class BotTurnRefusedError extends Error {
+  constructor(
+    readonly code: BotTurnRefusalCodeV1,
+    message: string,
+  ) {
+    super(message);
+    this.name = `${BOT_TURN_REFUSAL_PREFIX_V1}${code}`;
+  }
+}
+
+/** The refusal an error carries, or `undefined` when it is not one. */
+export function botTurnRefusalCodeV1(
+  error: unknown,
+): BotTurnRefusalCodeV1 | undefined {
+  const name =
+    typeof error === "object" && error !== null && "name" in error
+      ? String((error as { name: unknown }).name)
+      : "";
+  if (!name.startsWith(BOT_TURN_REFUSAL_PREFIX_V1)) return undefined;
+  const code = name.slice(BOT_TURN_REFUSAL_PREFIX_V1.length);
+  return code === "busy" ||
+    code === "reconciliation-required" ||
+    code === "fenced" ||
+    code === "duplicate"
+    ? code
+    : undefined;
+}
+
 export class BotTurnRecoveryRequiredError extends Error {
   constructor(readonly events: SessionEvent[]) {
     super("Bot turn has a durable outcome settlement pending");