@frockbot/kernel-do 0.3.7 → 0.3.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/kernel-do",
-  "version": "0.3.7",
+  "version": "0.3.8",
   "private": false,
   "type": "module",
   "exports": {
@@ -12,8 +12,8 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-composition": "0.3.7",
-    "@frockbot/kernel-contracts": "0.3.7",
+    "@frockbot/kernel-composition": "0.3.8",
+    "@frockbot/kernel-contracts": "0.3.8",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/authority.ts

@@ -12,6 +12,7 @@ import type { CompositionGenerationV1 } from "@frockbot/kernel-composition/gener
 import { DurableCompositionStore } from "./composition-store.js";
 import { DurableCompositionFailureLog } from "./composition-failures.js";
 import {
+  boundedRunFailureV1,
   botTurnCommandFingerprintV1,
   defaultRunLaneV1,
   storedRunAdmissionV1,
@@ -175,6 +176,16 @@ export const SUPERSEDED_TURN_REASON_V1 = "superseded by a new user message";
 /** How many times a queued Turn retries the object before giving up. */
 const MAX_QUEUED_RUN_START_ATTEMPTS = 8;
 
+/**
+ * The failure a discarded Turn is settled with when recovery finds it.
+ *
+ * It is never read by anybody: `failStoredRun` routes a run carrying a Stop or
+ * supersede intent to `cancelStoredRun`/`supersedeStoredRun`, and both drop the
+ * failure — the User's own intent is the outcome, not an error.
+ */
+const DISCARDED_RUN_RECOVERY_FAILURE_V1 =
+  "Turn was discarded before recovery could resume it";
+
 /**
  * True when this object has already durably decided to throw the Turn away.
  *
@@ -477,10 +488,43 @@ export class BotDurableAuthority<Snapshot> {
           `Reconciliation was explicitly abandoned: ${failure}`,
         );
       }
+      // "Try again" that ends in a settled run is a *successful* abandon, not a
+      // failed request. Rethrowing here made the button answer 409 and left the
+      // browser reading a run it thought had not moved — and the read that
+      // followed 500'd on the half-repaired record. The run is durable and
+      // terminal by this point, and its own record says why it ended, so the
+      // caller is handed that record and reads the reason from the transcript.
+      const settled = await this.settledReconciliationResult(runId);
+      if (settled) return settled;
       throw error;
     }
   }
 
+  /**
+   * The completion an abandoned reconciliation reports once the run it was
+   * resolving has reached a terminal state — whatever that state turned out to
+   * be. Anything still open is not this method's to answer for.
+   */
+  private async settledReconciliationResult(
+    runId: string,
+  ): Promise<BotTurnCompletion | undefined> {
+    const run = this.codec.optional(
+      await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
+    );
+    if (
+      run?.status !== "failed" &&
+      run?.status !== "cancelled" &&
+      run?.status !== "superseded"
+    ) {
+      return undefined;
+    }
+    return {
+      runId,
+      text: run.responseText ?? "",
+      events: structuredClone(run.events),
+    };
+  }
+
   private withNotification(
     snapshot: Snapshot,
     result: BotTurnCompletion,
@@ -572,7 +616,7 @@ export class BotDurableAuthority<Snapshot> {
         throw new Error(message);
       }
       await this.failRun(command.runId, previous, events, message);
-      const settled = await this.supersededRunResult(command.runId);
+      const settled = await this.discardedRunResult(command.runId);
       if (settled) return settled;
       throw new Error(message);
     } finally {
@@ -583,17 +627,25 @@ export class BotDurableAuthority<Snapshot> {
   }
 
   /**
-   * The completion a Turn another user message replaced reports. It is not a
-   * failure: the Turn settled durably, keeping everything it had already sent,
-   * and its caller reads the rest of the conversation from durable state.
+   * The completion a discarded Turn reports — one the User stopped, or one a
+   * later message replaced. Neither is a failure: the Turn settled durably,
+   * keeping everything it had already sent, and its caller reads the rest of
+   * the conversation from durable state.
+   *
+   * Stop used to be missing from here, so the long-lived `POST /turns` the
+   * composer was still holding open answered 500 the instant Stop was pressed:
+   * the UI said "You stopped this." and the console said the send had failed.
+   * A Turn the person stopped on purpose is the most ordinary outcome there is.
    */
-  private async supersededRunResult(
+  private async discardedRunResult(
     runId: string,
   ): Promise<BotTurnCompletion | undefined> {
     const run = this.codec.optional(
       await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
     );
-    if (run?.status !== "superseded") return undefined;
+    if (run?.status !== "superseded" && run?.status !== "cancelled") {
+      return undefined;
+    }
     return { runId, text: "", events: structuredClone(run.events) };
   }
 
@@ -608,7 +660,7 @@ export class BotDurableAuthority<Snapshot> {
     const run = this.codec.optional(
       await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
     );
-    if (run?.status === "superseded") {
+    if (run?.status === "superseded" || run?.status === "cancelled") {
       return { runId, text: "", events: structuredClone(run.events) };
     }
     if (run?.status !== "completed") return undefined;
@@ -705,7 +757,7 @@ export class BotDurableAuthority<Snapshot> {
         throw new Error(message);
       }
       await this.failRun(run.runId, previous, events, message);
-      const settled = await this.supersededRunResult(run.runId);
+      const settled = await this.discardedRunResult(run.runId);
       if (settled) return settled;
       throw new Error(message);
     } finally {
@@ -743,10 +795,11 @@ export class BotDurableAuthority<Snapshot> {
         `Turn idempotency key "${runId}" was reused for a different command`,
       );
     }
-    // A Turn another user message took the place of is an ordinary outcome,
-    // not a failure: it settled durably, said whatever it had already said,
-    // and the caller reads the rest from durable state.
-    if (run.status === "superseded") {
+    // A Turn the User stopped, or one another user message took the place of,
+    // is an ordinary outcome and not a failure: it settled durably, said
+    // whatever it had already said, and the caller reads the rest from durable
+    // state. A retry of either replays that settlement rather than refusing.
+    if (run.status === "superseded" || run.status === "cancelled") {
       return {
         runId,
         text: "",
@@ -881,7 +934,30 @@ export class BotDurableAuthority<Snapshot> {
         return;
       }
     }
-    await this.recoverActiveRun();
+    // An alarm has no caller. A rejection here is an uncaught exception in the
+    // object, and in the dev Worker it took the whole process down: a Stop left
+    // a run whose model outcome was uncertain, recovery re-entered it,
+    // `executeAdmittedRun` rethrew after recording the failure durably, and
+    // wrangler exited mid-run for every agent sharing the stack.
+    //
+    // Nothing about that throw is actionable here. Recovery has already written
+    // whatever it decided to durable storage before it rethrew, so the only
+    // thing left to do is record the reason and make sure the object still has
+    // a deadline — the re-arm is deliberately in a `finally`, because a failed
+    // recovery is exactly the case where the *next* firing matters most.
+    try {
+      await this.recoverActiveRun();
+    } catch (error) {
+      console.error(
+        `Bot run recovery alarm failed: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    } finally {
+      await this.ctx.storage
+        .transaction((transaction) => this.refreshRecoveryAlarm(transaction))
+        .catch(() => undefined);
+    }
   }
 
   /** Active run id, for Package projections of durable run state. */
@@ -1518,6 +1594,20 @@ export class BotDurableAuthority<Snapshot> {
     });
   }
 
+  /**
+   * Settles a run `failed` on a reason the authority composed from an error.
+   *
+   * The reason is bounded on the way in because nothing upstream bounds an
+   * error's `message`: a provider that echoes the request back produced one far
+   * past what the record allows, the settlement wrote it anyway, and every
+   * later read of that run threw — so a Turn that failed once went on to 500
+   * the transcript endpoint for ever. A reason a person reads loses nothing by
+   * being cut; a transcript nobody can read loses everything.
+   *
+   * Recovery's own `failStoredRun` is deliberately not routed through here: a
+   * failure derived from a malformed durable history is the one case where
+   * refusing to settle, and keeping the work active, is the right answer.
+   */
   private async failRun(
     runId: string,
     previous: SessionEvent[],
@@ -1532,13 +1622,14 @@ export class BotDurableAuthority<Snapshot> {
         runId,
         previous,
         events,
-        failure,
+        boundedRunFailureV1(failure),
         this.supersededPackageRecords(),
       );
       await this.refreshRecoveryAlarm(transaction);
     });
   }
 
+  /** Parks a run on a reason the authority composed, bounded as `failRun`'s is. */
   private async requireRunReconciliation(
     runId: string,
     previous: SessionEvent[],
@@ -1553,7 +1644,7 @@ export class BotDurableAuthority<Snapshot> {
         runId,
         previous,
         events,
-        failure,
+        boundedRunFailureV1(failure),
       );
       await this.refreshRecoveryAlarm(transaction);
     });
@@ -1631,6 +1722,30 @@ export class BotDurableAuthority<Snapshot> {
       const latest = (
         (await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
       ).map(decodeSessionEvent);
+      // A Turn the User stopped, or one a later message replaced, is terminal
+      // in intent before recovery ever looks at it. There is nothing to
+      // recover: no answer is owed, and the provider outcome cannot change what
+      // it settles as. Re-entering it is how the Worker died — the run resumed,
+      // reached "Model response outcome is uncertain after cancellation", and
+      // the alarm had nothing to hand the rejection to.
+      //
+      // `failStoredRun` routes a discarded run to `cancelStoredRun` or
+      // `supersedeStoredRun` on the intent that is already durable, and closes
+      // the open turn on the way, so the settled log is a complete account.
+      if (runWasDiscardedV1(run)) {
+        await failStoredRun(
+          this.codec,
+          transaction,
+          this.terminalKeys(run.runId),
+          run.runId,
+          latest.slice(0, run.previousEventCount),
+          run.events,
+          DISCARDED_RUN_RECOVERY_FAILURE_V1,
+          this.supersededPackageRecords(),
+        );
+        await this.refreshRecoveryAlarm(transaction);
+        return undefined;
+      }
       const plan = planBotRunRecovery(
         run,
         latest,
@@ -1707,22 +1822,6 @@ export class BotDurableAuthority<Snapshot> {
         await this.refreshRecoveryAlarm(transaction);
         return { kind: "resume" as const, run, latest, settings };
       }
-      if (runWasDiscardedV1(run)) {
-        // Recovery of a Turn Stop or supersede already discarded settles it on
-        // that intent rather than parking it: nothing is owed the answer.
-        await failStoredRun(
-          this.codec,
-          transaction,
-          this.terminalKeys(run.runId),
-          run.runId,
-          latest.slice(0, run.previousEventCount),
-          [...run.events, ...plan.repairs],
-          "Execution outcome requires reconciliation before it can resume",
-          this.supersededPackageRecords(),
-        );
-        await this.refreshRecoveryAlarm(transaction);
-        return undefined;
-      }
       await transaction.put({
         [key]: {
           ...run,

package/src/run-failure-bounds.test.ts

@@ -0,0 +1,46 @@
+// A failure the record could not hold, and the transcript it took with it.
+//
+// A run's `failure` is whatever an error's `message` happened to be, and
+// nothing upstream bounds it. A provider that echoed the request back produced
+// one past the record's own limit, the settlement wrote it anyway, and every
+// later read of that run threw `has invalid failure` — so a Turn that failed
+// once went on to 500 the transcript endpoint for ever, and "Try again" could
+// not get the person out of it either.
+import { describe, expect, test } from "bun:test";
+import {
+  boundedRunFailureV1,
+  MAX_RUN_FAILURE_BYTES_V1,
+} from "./run-records.ts";
+
+const bytes = (value: string) => new TextEncoder().encode(value).byteLength;
+
+describe("the failure a settlement may durably write", () => {
+  test("leaves a reason that already fits exactly as it was written", () => {
+    const reason = "Reconciliation was explicitly abandoned: response lost";
+    expect(boundedRunFailureV1(reason)).toBe(reason);
+  });
+
+  test("cuts one the record could not hold, and says it was cut", () => {
+    const bounded = boundedRunFailureV1(
+      "x".repeat(MAX_RUN_FAILURE_BYTES_V1 * 2),
+    );
+    expect(bytes(bounded)).toBeLessThanOrEqual(MAX_RUN_FAILURE_BYTES_V1);
+    expect(bounded.endsWith("…")).toBe(true);
+  });
+
+  test("keeps the opening words, which are the ones a person reads", () => {
+    const bounded = boundedRunFailureV1(
+      `Model request failed: ${"detail ".repeat(MAX_RUN_FAILURE_BYTES_V1)}`,
+    );
+    expect(bounded.startsWith("Model request failed: detail")).toBe(true);
+  });
+
+  test("never cuts inside a character", () => {
+    // Multi-byte throughout: a byte-wise slice would leave a lone surrogate
+    // and the decoder would refuse the record for a different reason.
+    const bounded = boundedRunFailureV1("🐑".repeat(MAX_RUN_FAILURE_BYTES_V1));
+    expect(bytes(bounded)).toBeLessThanOrEqual(MAX_RUN_FAILURE_BYTES_V1);
+    expect(bounded).not.toContain("�");
+    expect([...bounded].every((character) => character.length <= 2)).toBe(true);
+  });
+});

package/src/run-records.ts

@@ -277,6 +277,36 @@ function boundedString(
   );
 }
 
+/** The largest failure a run record may carry, in UTF-8 bytes. */
+export const MAX_RUN_FAILURE_BYTES_V1 = 8_000;
+
+/**
+ * The failure a settlement may durably write, cut to what the record allows.
+ *
+ * A failure string is whatever an error's `message` happened to be, and nothing
+ * upstream bounds it — a provider that echoes a request back, or a message
+ * built by concatenating one, easily runs past the limit. Writing it anyway
+ * produced a record the codec refused on every later read, so a Turn that
+ * failed once went on to 500 the transcript endpoint for ever. The message is
+ * for a person to read: cutting it costs nothing the record does not already
+ * hold, and losing the whole transcript costs everything.
+ */
+export function boundedRunFailureV1(failure: string): string {
+  if (UTF8_ENCODER.encode(failure).byteLength <= MAX_RUN_FAILURE_BYTES_V1) {
+    return failure;
+  }
+  const ellipsis = "…";
+  const budget =
+    MAX_RUN_FAILURE_BYTES_V1 - UTF8_ENCODER.encode(ellipsis).byteLength;
+  let kept = failure;
+  // Cutting by characters and re-measuring keeps the result valid UTF-8; a
+  // byte-wise slice can land inside a multi-byte sequence.
+  while (UTF8_ENCODER.encode(kept).byteLength > budget) {
+    kept = kept.slice(0, Math.max(0, Math.floor(kept.length * 0.9) - 1));
+  }
+  return `${kept}${ellipsis}`;
+}
+
 function decodeDirectToolCommandV1(value: unknown): DirectToolCommandV1 {
   if (!value || typeof value !== "object" || Array.isArray(value)) {
     throw new Error("stored run has invalid direct tool command");

package/src/turn-supersede.test.ts

@@ -911,3 +911,125 @@ describe("an interrupt while the model is streaming", () => {
     expect((await next).text).toBe("done: second");
   });
 });
+
+describe("a discarded Turn never crashes the object", () => {
+  test("the long-lived caller is answered with the cancelled run, not a throw", async () => {
+    const storage = new MemoryStorage();
+    const probe = createAuthority(storage, { uncertain: () => true });
+
+    const first = probe.authority.run(command("run-1", "first"));
+    await probe.handle("run-1").started;
+    const stopped = storedRun(storage, "run-1");
+    storage.values.set("run:run-1", {
+      ...stopped,
+      stopRequestedAt: "2026-09-03T00:00:05.000Z",
+    });
+    probe.handle("run-1").interrupt("agent cancelled by user");
+
+    // The composer is still holding this request open when Stop is pressed. It
+    // used to be answered with a 500 and a red console error while the UI
+    // beside it said "You stopped this." A Turn the person stopped on purpose
+    // is an ordinary outcome and settles as one.
+    const settled = await first;
+    expect(settled.runId).toBe("run-1");
+    expect(settled.text).toBe("");
+    expect(storedRun(storage, "run-1").status).toBe("cancelled");
+    expect(storage.values.get("active-run")).toBeUndefined();
+  });
+
+  test("recovery settles a stopped Turn instead of re-entering it", async () => {
+    const storage = new MemoryStorage();
+    const probe = createAuthority(storage, { uncertain: () => true });
+
+    const first = probe.authority.run(command("run-1", "first"));
+    await probe.handle("run-1").started;
+    const running = storedRun(storage, "run-1");
+    storage.values.set("run:run-1", {
+      ...running,
+      stopRequestedAt: "2026-09-03T00:00:05.000Z",
+    });
+    // The object is evicted with the Stop durable and the Turn still active:
+    // exactly the state the recovery alarm wakes up to.
+    first.catch(() => undefined);
+
+    const evicted = createAuthority(storage, { uncertain: () => true });
+    await expect(evicted.authority.alarm()).resolves.toBeUndefined();
+
+    // Re-entering it is what took the dev Worker down: the run resumed, reached
+    // "Model response outcome is uncertain after cancellation", and the alarm
+    // had nobody to hand the rejection to. There is nothing to recover — the
+    // User already said to throw it away.
+    expect(evicted.observed).toEqual([]);
+    expect(storedRun(storage, "run-1").status).toBe("cancelled");
+    expect(storage.values.get("active-run")).toBeUndefined();
+  });
+
+  test("recovery settles a superseded Turn instead of re-entering it", async () => {
+    const storage = new MemoryStorage();
+    const probe = createAuthority(storage, { uncertain: () => true });
+
+    const first = probe.authority.run(command("run-1", "first"));
+    await probe.handle("run-1").started;
+    const running = storedRun(storage, "run-1");
+    storage.values.set("run:run-1", {
+      ...running,
+      supersededAt: "2026-09-03T00:00:05.000Z",
+      supersededBy: "run-2",
+    });
+    first.catch(() => undefined);
+
+    const evicted = createAuthority(storage, { uncertain: () => true });
+    await expect(evicted.authority.alarm()).resolves.toBeUndefined();
+
+    expect(evicted.observed).toEqual([]);
+    expect(storedRun(storage, "run-1").status).toBe("superseded");
+    expect(storage.values.get("active-run")).toBeUndefined();
+  });
+
+  test("an alarm records a recovery failure instead of rejecting", async () => {
+    const storage = new MemoryStorage();
+    const probe = createAuthority(storage, { failRecovery: () => true });
+
+    const first = probe.authority.run(command("run-1", "first"));
+    await probe.handle("run-1").started;
+    first.catch(() => undefined);
+
+    // A recovery that cannot run the Turn is a durable fact, not a fault of the
+    // alarm: an alarm has no caller, so anything it lets escape is an uncaught
+    // exception in the object — one of the ways the dev Worker died.
+    const evicted = createAuthority(storage, { failRecovery: () => true });
+    await expect(evicted.authority.alarm()).resolves.toBeUndefined();
+    // And the object still has a deadline, so the next firing tries again.
+    expect(storage.alarmAt).toBeGreaterThan(0);
+  });
+});
+
+describe("Try again on a parked Turn", () => {
+  test("answers with the run it settled rather than throwing", async () => {
+    const storage = new MemoryStorage();
+    // The Turn parks on a provider outcome only a User can retrieve, which is
+    // the state the Resolve Turn button exists for.
+    const probe = createAuthority(storage, {
+      dispatch: () => false,
+      parkOnRelease: () => true,
+    });
+
+    const first = probe.authority.run(command("run-1", "first"));
+    await probe.handle("run-1").started;
+    probe.handle("run-1").finish();
+    await first.catch(() => undefined);
+    expect(storedRun(storage, "run-1").status).toBe("reconciliation-required");
+
+    // "Try again": the retry fails again, the run is abandoned, and that is a
+    // successful abandon — not a failed request. Rethrowing here made the
+    // button answer 409, and the transcript read the browser makes straight
+    // afterwards 500 on the half-repaired record.
+    const abandoned = await probe.authority.reconcileRun(identity, "run-1");
+    expect(abandoned.runId).toBe("run-1");
+
+    const settled = storedRun(storage, "run-1");
+    expect(settled.status).toBe("failed");
+    expect(settled.failure).toContain("explicitly abandoned");
+    expect(storage.values.get("active-run")).toBeUndefined();
+  });
+});