@frockbot/plugin-routines 0.3.3 → 0.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/plugin-routines",
-  "version": "0.3.3",
+  "version": "0.3.6",
   "private": false,
   "type": "module",
   "exports": {
@@ -32,12 +32,12 @@
     "typecheck": "vue-tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/client-core": "0.3.3",
-    "@frockbot/client-ui": "0.3.3",
-    "@frockbot/configuration-core": "0.3.3",
-    "@frockbot/kernel-agent-loop": "0.3.3",
-    "@frockbot/kernel-contracts": "0.3.3",
-    "@frockbot/plugin-shell": "0.3.3",
+    "@frockbot/client-core": "0.3.6",
+    "@frockbot/client-ui": "0.3.6",
+    "@frockbot/configuration-core": "0.3.6",
+    "@frockbot/kernel-agent-loop": "0.3.6",
+    "@frockbot/kernel-contracts": "0.3.6",
+    "@frockbot/plugin-shell": "0.3.6",
     "cordis": "4.0.0-rc.8",
     "croner": "10.0.1",
     "vue": "3.5.41"

package/src/client/RoutineInboxBadge.vue

@@ -69,8 +69,7 @@ function acknowledge(entryIds: string[]): void {
         >
       </header>
       <p v-if="routines.inbox.length === 0" class="routine-inbox__empty">
-        Nothing has been left here. A Routine that finishes lands its message in
-        this drawer rather than in the conversation.
+        Nothing here yet. Finished Routines leave their results here.
       </p>
       <ul v-else class="routine-inbox__list">
         <li

package/src/client/RoutinesSection.vue

@@ -158,8 +158,8 @@ async function toggleLog(routineId: string): Promise<void> {
       <span class="routines__intro">
         <strong>Routines</strong>
         <small>
-          Standing instructions this Bot runs on a schedule or on a delivered
-          webhook, as their own Turns.
+          Standing instructions this Bot runs on a schedule, or when a webhook
+          fires.
         </small>
       </span>
       <UiButton type="button" :disabled="!botId" @click="startCreate">
@@ -175,8 +175,7 @@ async function toggleLog(routineId: string): Promise<void> {
       v-if="routines.loaded && routines.routines.length === 0"
       class="routines__empty"
     >
-      No Routines yet. A Routine fires on its own and reports back on this Bot's
-      next conversation.
+      No Routines yet. Set one up to have this Bot do something on a schedule.
     </p>
 
     <article
@@ -204,8 +203,8 @@ async function toggleLog(routineId: string): Promise<void> {
           >Webhook key, version {{ routines.mintedHook.keyVersion }}</strong
         >
         <small>
-          This is the only time it is shown. Copy it now; rotating is the only
-          way to see a key again.
+          This is the only time you'll see this key. Copy it now — you'll need a
+          new one otherwise.
         </small>
         <code>{{ hookUrl() }}</code>
         <code class="routine-hook__token">{{ routines.mintedHook.token }}</code>

package/src/firing.ts

@@ -36,6 +36,14 @@ export interface RoutineScheduleStateV1 {
   dueAt: number;
   /** Epoch milliseconds before which the alarm must not settle this Routine. */
   deferredUntil?: number;
+  /**
+   * Firings that failed in a row, reset by the first that did not.
+   *
+   * It is on the clock rather than on the record because it is timing state:
+   * what it buys is the backoff `deferredUntil` carries, and the auto-pause it
+   * ends in.
+   */
+  consecutiveFailures?: number;
 }
 
 /** One firing, durable before the Turn it admits. The same-Routine lock. */
@@ -92,7 +100,7 @@ export function decodeRoutineScheduleStateV1(
   routineExactKeys(
     candidate,
     ["schemaVersion", "routineId", "anchor", "dueAt"],
-    ["deferredUntil"],
+    ["deferredUntil", "consecutiveFailures"],
     "Routine schedule state",
   );
   if (candidate.schemaVersion !== 1) {
@@ -116,6 +124,14 @@ export function decodeRoutineScheduleStateV1(
             "Routine schedule state deferredUntil",
           ),
         }),
+    ...(candidate.consecutiveFailures === undefined
+      ? {}
+      : {
+          consecutiveFailures: count(
+            candidate.consecutiveFailures,
+            "Routine schedule state consecutiveFailures",
+          ),
+        }),
   };
 }
 

package/src/scheduler.test.ts

@@ -6,12 +6,17 @@ import {
 } from "./scheduler.js";
 import { decodeRoutineScheduleStateV1, type RoutineFireV1 } from "./firing.js";
 import { RoutineStore } from "./store.js";
+import { RoutineInboxStore } from "./inbox-store.js";
 import { createMemoryRoutineStorageV1 } from "./testing.js";
 import {
   routineFireKeyV1,
   routineScheduleKeyV1,
+  ROUTINE_FAILURE_PAUSE_AFTER,
+  ROUTINE_FIRE_LEASE_MS,
   ROUTINE_QUEUE_LIMIT,
+  ROUTINE_RUN_PREFIX,
 } from "./storage-keys.js";
+import { decodeRoutineRunEntryV1 } from "./records.js";
 import type { RoutineCommandV1 } from "./shared.js";
 
 const USER = { kind: "user" } as const;
@@ -30,10 +35,23 @@ function clock(start: string) {
   };
 }
 
-function harness(options: { start: string; schedule?: string }) {
+function harness(options: {
+  start: string;
+  schedule?: string;
+  fireTimeoutMs?: number;
+  fireLeaseMs?: number;
+}) {
   const storage = createMemoryRoutineStorageV1();
   const time = clock(options.start);
-  const scheduler = new RoutineScheduler(storage, { now: time.now });
+  const scheduler = new RoutineScheduler(storage, {
+    now: time.now,
+    ...(options.fireTimeoutMs === undefined
+      ? {}
+      : { fireTimeoutMs: options.fireTimeoutMs }),
+    ...(options.fireLeaseMs === undefined
+      ? {}
+      : { fireLeaseMs: options.fireLeaseMs }),
+  });
   const store = new RoutineStore(storage, {
     now: time.now,
     firings: scheduler,
@@ -480,3 +498,293 @@ describe("nextRuns", () => {
     ).toBeUndefined();
   });
 });
+
+describe("an abandoned firing", () => {
+  /** Every run-log entry stored, newest first. */
+  async function runLog(
+    storage: ReturnType<typeof createMemoryRoutineStorageV1>,
+  ) {
+    const stored = await storage.list<unknown>({ prefix: ROUTINE_RUN_PREFIX });
+    return [...stored.values()].map((value) => decodeRoutineRunEntryV1(value));
+  }
+
+  test("still contributes a deadline, so the object's alarm is not deleted under it", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T08:59:00.000Z",
+      schedule: "0 9 * * *",
+    });
+    await store.execute(create, USER);
+
+    // The isolate is killed between `#writeClaim` and `#settleFiring`: the
+    // lock is durable and nothing will ever delete it.
+    time.set("2026-01-01T09:00:00.000Z");
+    await scheduler.settle(async () => {
+      throw new DOMException("CPU time limit", "Error");
+    });
+    // (that one settles) — now stage a genuinely orphaned lock.
+    await storage.put(routineFireKeyV1("brief"), {
+      schemaVersion: 1,
+      routineId: "brief",
+      fireId: "rf-brief-orphan",
+      trigger: "cron",
+      cue: "Routine fired.",
+      mintedAt: "2026-01-01T09:00:00.000Z",
+      entryId: "rf-brief-orphan-entry",
+    });
+
+    // Before the lease: the Routine is being dealt with, and the deadline the
+    // object arms on is when the lease runs out — never nothing, which is what
+    // let `deleteAlarm()` silence the Bot.
+    time.set("2026-01-01T09:01:00.000Z");
+    expect(await scheduler.deadlines(storage)).toEqual([
+      Date.parse("2026-01-01T09:00:00.000Z") + ROUTINE_FIRE_LEASE_MS,
+    ]);
+  });
+
+  test("is reaped once its lease expires, and the Routine fires again", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T08:59:00.000Z",
+      schedule: "0 9 * * *",
+    });
+    await store.execute(create, USER);
+    await storage.put(routineFireKeyV1("brief"), {
+      schemaVersion: 1,
+      routineId: "brief",
+      fireId: "rf-brief-orphan",
+      trigger: "cron",
+      cue: "Routine fired.",
+      mintedAt: "2026-01-01T09:00:00.000Z",
+      entryId: "rf-brief-orphan-entry",
+    });
+
+    time.set(
+      new Date(
+        Date.parse("2026-01-01T09:00:00.000Z") + ROUTINE_FIRE_LEASE_MS + 1,
+      ).toISOString(),
+    );
+    const fired = await drain(scheduler);
+
+    // The lock is gone, the abandoned firing is `failed` in the log with a
+    // reason, and the next occurrence really ran.
+    expect(await storage.get(routineFireKeyV1("brief"))).toBeUndefined();
+    const log = await runLog(storage);
+    expect(
+      log.find((entry) => entry.fireId === "rf-brief-orphan"),
+    ).toMatchObject({ status: "failed" });
+    expect(fired).toHaveLength(1);
+  });
+
+  test("releases a lock nothing can decode", async () => {
+    const { storage, scheduler, store, create } = harness({
+      start: "2026-01-01T08:59:00.000Z",
+      schedule: "0 9 * * *",
+    });
+    await store.execute(create, USER);
+    await storage.put(routineFireKeyV1("brief"), { schemaVersion: 99 });
+
+    await scheduler.reapExpiredFirings();
+
+    expect(await storage.get(routineFireKeyV1("brief"))).toBeUndefined();
+  });
+});
+
+describe("a firing that never comes back", () => {
+  test("is stopped at its timeout and settled as failed", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T08:59:00.000Z",
+      schedule: "0 9 * * *",
+      fireTimeoutMs: 5,
+    });
+    await store.execute(create, USER);
+
+    time.set("2026-01-01T09:00:00.000Z");
+    let aborted = false;
+    await scheduler.settle(
+      (_fire, signal) =>
+        new Promise((resolve) => {
+          signal.addEventListener("abort", () => {
+            aborted = true;
+            // Never resolves on its own: the timeout is the only thing that
+            // ends this firing.
+          });
+          void resolve;
+        }),
+    );
+
+    expect(aborted).toBe(true);
+    expect(await storage.get(routineFireKeyV1("brief"))).toBeUndefined();
+    const stored = await storage.list<unknown>({ prefix: ROUTINE_RUN_PREFIX });
+    const entries = [...stored.values()].map((value) =>
+      decodeRoutineRunEntryV1(value),
+    );
+    expect(entries[0]).toMatchObject({ status: "failed" });
+    expect(entries[0]?.summary).toContain("longer than");
+  });
+});
+
+describe("a firing that keeps failing", () => {
+  test("backs off instead of re-arming identically, and pauses itself", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T00:00:00.000Z",
+      schedule: "@every 1m",
+    });
+    await store.execute(create, USER);
+
+    // The live wedge was `ok, ok, ok, failed, failed, …` at exactly 60s
+    // intervals, for ever: every failure re-armed the next occurrence
+    // identically and nothing paused, backed off, or said so.
+    const failing = async () =>
+      scheduler.settle(async () => {
+        firedAt.push(time.now().getTime());
+        return { status: "failed", summary: "the provider refused" };
+      });
+    const firedAt: number[] = [];
+
+    time.set("2026-01-01T00:01:00.000Z");
+    await failing();
+    time.set("2026-01-01T00:02:00.000Z");
+    await failing();
+    expect(firedAt).toHaveLength(2);
+    // The third occurrence is due — and the backoff holds it, where before
+    // every failure re-armed the next occurrence identically.
+    time.set("2026-01-01T00:03:00.000Z");
+    await failing();
+    expect(firedAt).toHaveLength(2);
+
+    // Given all the time in the world it still fires only until it gives up.
+    for (let step = 1; step <= 10; step += 1) {
+      time.set(
+        new Date(
+          Date.parse("2026-01-01T00:02:00.000Z") + step * 60 * 60_000,
+        ).toISOString(),
+      );
+      await failing();
+    }
+    expect(firedAt).toHaveLength(ROUTINE_FAILURE_PAUSE_AFTER);
+    const listed = await store.list("scout", await scheduler.nextRuns());
+    expect(listed.routines[0]).toMatchObject({ enabled: false });
+    // The reason is durable and in the run log the panel already renders.
+    const runs = await storage.list<unknown>({ prefix: ROUTINE_RUN_PREFIX });
+    const paused = [...runs.values()]
+      .map((value) => decodeRoutineRunEntryV1(value))
+      .find((entry) => entry.summary?.includes("times in a row"));
+    expect(paused).toMatchObject({ status: "skipped" });
+  });
+
+  test("a firing that succeeds clears the backoff", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T00:00:00.000Z",
+      schedule: "@every 1m",
+    });
+    await store.execute(create, USER);
+
+    time.set("2026-01-01T00:01:00.000Z");
+    await drain(scheduler, { status: "failed", summary: "flaked" });
+    expect((await state(storage)).consecutiveFailures).toBe(1);
+
+    time.set("2026-01-01T00:05:00.000Z");
+    await drain(scheduler);
+    expect((await state(storage)).consecutiveFailures).toBeUndefined();
+  });
+});
+
+describe("a burst of missed occurrences", () => {
+  test("coalesces into one firing even inside the grace window", async () => {
+    const { time, scheduler, store, create } = harness({
+      start: "2026-01-01T00:00:00.000Z",
+      schedule: "@every 1m",
+    });
+    await store.execute(create, USER);
+
+    // Four minutes of stall — well inside the five-minute grace, which is
+    // exactly where each occurrence used to be claimed on its own and one
+    // drain ran four Turns back to back.
+    time.set("2026-01-01T00:04:30.000Z");
+    const fired = await drain(scheduler);
+
+    expect(fired).toHaveLength(1);
+    expect(fired[0]?.missedCount).toBe(4);
+    expect(fired[0]?.cue).toContain("4 scheduled occurrences elapsed");
+  });
+});
+
+describe("a failed firing", () => {
+  test("writes a completion-inbox entry the person can see", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T00:00:00.000Z",
+      schedule: "@every 1m",
+    });
+    await store.execute(create, USER);
+
+    time.set("2026-01-01T00:01:00.000Z");
+    await drain(scheduler, {
+      status: "failed",
+      summary: "turn 10 started while turn 9 is open",
+    });
+
+    // Before this, a failed firing produced only a `failed` run-log row: the
+    // header badge stayed at its old count through six consecutive failures
+    // and the only trace was behind Run log → expand a row.
+    const inbox = new RoutineInboxStore(storage, { now: time.now });
+    const entries = await inbox.list();
+    expect(entries).toHaveLength(1);
+    expect(entries[0]).toMatchObject({
+      routineId: "brief",
+      acknowledged: false,
+    });
+    expect(entries[0]?.text).toContain("Morning brief");
+    expect(entries[0]?.text).toContain("turn 10 started while turn 9 is open");
+    // A failure is addressed to the person, not handed to the Bot's next
+    // conversational Turn: no pending wake.
+    expect(await inbox.pending()).toEqual([]);
+  });
+
+  test("records one entry however many times the settle is retried", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T00:00:00.000Z",
+      schedule: "@every 1m",
+    });
+    await store.execute(create, USER);
+    time.set("2026-01-01T00:01:00.000Z");
+    await drain(scheduler, { status: "failed", summary: "flaked" });
+    time.set("2026-01-01T02:00:00.000Z");
+    await drain(scheduler, { status: "failed", summary: "flaked" });
+
+    const entries = await new RoutineInboxStore(storage, {
+      now: time.now,
+    }).list();
+    expect(new Set(entries.map((entry) => entry.entryId)).size).toBe(
+      entries.length,
+    );
+  });
+});
+
+describe("an undecodable Routine record", () => {
+  test("degrades that one Routine, never the whole object", async () => {
+    const { storage, time, scheduler, store, create } = harness({
+      start: "2026-01-01T08:59:00.000Z",
+      schedule: "0 9 * * *",
+    });
+    await store.execute(create, USER);
+    // A record written by a newer deploy and read back after a rollback: the
+    // decoder is exact-keys, so it refuses it outright.
+    await storage.put("routine:from-the-future", {
+      schemaVersion: 1,
+      routineId: "from-the-future",
+      unknownFieldFromANewerDeploy: true,
+    });
+
+    // `#clocks` is reached from `deadlines()` → `refreshRecoveryAlarm`, which
+    // runs inside `completeRun`, `failRun` and `acceptRun`: a throw here used
+    // to poison every alarm refresh and every Turn settlement of the object.
+    time.set("2026-01-01T09:00:00.000Z");
+    expect(await scheduler.deadlines(storage)).toEqual([
+      Date.parse("2026-01-01T09:00:00.000Z"),
+    ]);
+    expect(await scheduler.nextRuns()).toEqual(
+      new Map([["brief", "2026-01-01T09:00:00.000Z"]]),
+    );
+    // And the healthy Routine still fires.
+    expect(await drain(scheduler)).toHaveLength(1);
+  });
+});

package/src/scheduler.ts

@@ -35,6 +35,7 @@ import {
   type RoutineFireV1,
   type RoutineScheduleStateV1,
 } from "./firing.js";
+import { routineTerminalRecordsV1 } from "./inbox-store.js";
 import {
   decodeRoutineRecordV1,
   type RoutineRecordV1,
@@ -50,6 +51,11 @@ import {
 import {
   nextQueueSequenceV1,
   ROUTINE_DEFERRAL_MS,
+  ROUTINE_FIRE_LEASE_MS,
+  ROUTINE_FIRE_PREFIX,
+  ROUTINE_FAILURE_BACKOFF_MS,
+  ROUTINE_FAILURE_PAUSE_AFTER,
+  ROUTINE_FIRE_TIMEOUT_MS,
   ROUTINE_LIMIT_PER_BOT,
   ROUTINE_MISSED_GRACE_MS,
   ROUTINE_PREFIX,
@@ -78,12 +84,34 @@ export interface RoutineFireOutcomeV1 {
  */
 export type RoutineFireExecutorV1 = (
   fire: RoutineFireV1,
+  signal: AbortSignal,
 ) => Promise<RoutineFireOutcomeV1>;
 
 export interface RoutineSchedulerOptionsV1 {
   now?(): Date;
   /** Injected so a test can watch the drain without a Durable Object. */
-  onSettled?(fire: RoutineFireV1, outcome: RoutineFireOutcomeV1): void;
+  onSettled?(
+    fire: RoutineFireV1,
+    outcome: RoutineFireOutcomeV1,
+  ): void | Promise<void>;
+  /** How long one firing's Turn may run. Overridden only by tests. */
+  fireTimeoutMs?: number;
+  /** How long an unsettled firing holds its lock. Overridden only by tests. */
+  fireLeaseMs?: number;
+}
+
+/**
+ * When an unsettled firing stops being "in flight" and starts being abandoned.
+ *
+ * Derived from `mintedAt`, which is already durable on the firing, so the lease
+ * needs no second record and survives the eviction it exists to survive.
+ */
+export function routineFireLeaseExpiryV1(
+  fire: RoutineFireV1,
+  leaseMs: number = ROUTINE_FIRE_LEASE_MS,
+): number {
+  const minted = Date.parse(fire.mintedAt);
+  return (Number.isFinite(minted) ? minted : 0) + leaseMs;
 }
 
 /** A firing waiting behind an unsettled one. */
@@ -156,7 +184,13 @@ export class RoutineScheduler {
   readonly #storage: RoutineStorageV1;
   readonly #now: () => Date;
   readonly #onSettled:
-    ((fire: RoutineFireV1, outcome: RoutineFireOutcomeV1) => void) | undefined;
+    | ((
+        fire: RoutineFireV1,
+        outcome: RoutineFireOutcomeV1,
+      ) => void | Promise<void>)
+    | undefined;
+  readonly #fireTimeoutMs: number;
+  readonly #fireLeaseMs: number;
 
   constructor(
     storage: RoutineStorageV1,
@@ -165,6 +199,8 @@ export class RoutineScheduler {
     this.#storage = storage;
     this.#now = options.now ?? (() => new Date());
     this.#onSettled = options.onSettled;
+    this.#fireTimeoutMs = options.fireTimeoutMs ?? ROUTINE_FIRE_TIMEOUT_MS;
+    this.#fireLeaseMs = options.fireLeaseMs ?? ROUTINE_FIRE_LEASE_MS;
   }
 
   /**
@@ -179,7 +215,23 @@ export class RoutineScheduler {
       const locked = await reads.get<unknown>(
         routineFireKeyV1(record.routineId),
       );
-      if (locked) continue;
+      if (locked) {
+        // A Routine with an unsettled firing is being dealt with — but only
+        // until its lease runs out. Skipping it outright is what let a firing
+        // killed mid-flight take the object's whole alarm down with it: the
+        // locked Routine contributed no deadline, so the kernel's
+        // `deleteAlarm()` ran and the Bot went silent for ever. The lease
+        // expiry is the deadline, and `settle` is what reaps it.
+        // The hold still applies, for the same reason it applies to a debt: an
+        // expired lease on a Routine whose Turn is executing right now would
+        // otherwise arm the alarm on a moment already past, over and over,
+        // which is a spin rather than a deadline. `defer()` moves the hold and
+        // never the lease, so the reaping still happens — just not on a loop.
+        deadlines.push(
+          Math.max(this.#leaseExpiry(locked), state.deferredUntil ?? 0),
+        );
+        continue;
+      }
       deadlines.push(routineDeadlineV1(state));
     }
     for (const routineId of await this.#queuedRoutineIds(reads)) {
@@ -209,20 +261,98 @@ export class RoutineScheduler {
    * durable firing, run it, settle it, then look again.
    */
   async settle(execute: RoutineFireExecutorV1): Promise<void> {
+    await this.reapExpiredFirings();
     for (let drained = 0; drained < ROUTINE_SETTLE_BATCH; drained += 1) {
       const claimed = await this.#claim();
       if (!claimed) return;
-      let outcome: RoutineFireOutcomeV1;
-      try {
-        outcome = await execute(claimed.fire);
-      } catch (error) {
-        outcome = {
+      const outcome = await this.#execute(execute, claimed.fire);
+      await this.#settleFiring(claimed.fire, outcome);
+      await this.#onSettled?.(claimed.fire, outcome);
+    }
+  }
+
+  /**
+   * Run one firing under a hard time bound.
+   *
+   * Nothing else bounds it: `maxSteps` bounds the loop and not the wall clock,
+   * and an automation run is not offered to the user's Stop. A firing that
+   * never comes back must still settle, or its lock outlives the isolate.
+   */
+  async #execute(
+    execute: RoutineFireExecutorV1,
+    fire: RoutineFireV1,
+  ): Promise<RoutineFireOutcomeV1> {
+    const controller = new AbortController();
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const expiry = new Promise<RoutineFireOutcomeV1>((resolve) => {
+      timer = setTimeout(() => {
+        controller.abort();
+        resolve({
           status: "failed",
-          summary: error instanceof Error ? error.message : String(error),
-        };
+          summary: `The Routine ran for longer than ${Math.round(
+            this.#fireTimeoutMs / 1000,
+          )} seconds and was stopped.`,
+        });
+      }, this.#fireTimeoutMs);
+    });
+    try {
+      return await Promise.race([
+        // `Promise.resolve().then` and not a bare call: an executor that throws
+        // synchronously must reach the same failed outcome as one that rejects,
+        // or the throw escapes `settle` with the fire lock still held.
+        Promise.resolve()
+          .then(() => execute(fire, controller.signal))
+          .catch((error: unknown): RoutineFireOutcomeV1 => ({
+            status: "failed",
+            summary: error instanceof Error ? error.message : String(error),
+          })),
+        expiry,
+      ]);
+    } finally {
+      if (timer !== undefined) clearTimeout(timer);
+    }
+  }
+
+  /**
+   * Settle for dead every firing whose lease has run out.
+   *
+   * The other half of the lease: `deadlines()` makes an abandoned firing wake
+   * the object, and this is what the object then does about it. An undecodable
+   * lock is released too — it names a firing nothing can run, and leaving it
+   * wedges the Routine exactly as an abandoned one does.
+   */
+  async reapExpiredFirings(): Promise<void> {
+    const now = this.#now().getTime();
+    const stored = await this.#storage.list<unknown>({
+      prefix: ROUTINE_FIRE_PREFIX,
+    });
+    for (const [key, value] of stored.entries()) {
+      let fire: RoutineFireV1;
+      try {
+        fire = decodeRoutineFireV1(value);
+      } catch {
+        await this.#storage.delete(key);
+        continue;
       }
-      await this.#settleFiring(claimed.fire, outcome);
-      this.#onSettled?.(claimed.fire, outcome);
+      if (this.#leaseExpiry(value) > now) continue;
+      await this.#settleFiring(fire, {
+        status: "failed",
+        summary:
+          "The Routine stopped without reporting an outcome and its firing was settled as failed.",
+      });
+    }
+  }
+
+  #leaseExpiry(stored: unknown): number {
+    try {
+      return routineFireLeaseExpiryV1(
+        decodeRoutineFireV1(stored),
+        this.#fireLeaseMs,
+      );
+    } catch {
+      // An undecodable lock is already overdue: reaping it is the only way its
+      // Routine ever fires again.
+      return 0;
     }
   }
 
@@ -327,7 +457,18 @@ export class RoutineScheduler {
       state: RoutineScheduleStateV1;
     }> = [];
     for (const value of stored.values()) {
-      const record = decodeRoutineRecordV1(value);
+      let record: RoutineRecordV1;
+      try {
+        record = decodeRoutineRecordV1(value);
+      } catch {
+        // One record written by a newer deploy and read back after a rollback
+        // used to poison every alarm refresh and every Turn settlement of the
+        // whole object: `#clocks` is reached from `deadlines()` →
+        // `refreshRecoveryAlarm`, which runs inside `completeRun`, `failRun`
+        // and `acceptRun`. A record nothing can read is one Routine that does
+        // not fire, never a Bot that cannot settle a Turn.
+        continue;
+      }
       if (!record.enabled || record.schedule === undefined) continue;
       const normalized = normalizeRoutineScheduleV1(
         record.schedule,
@@ -416,7 +557,14 @@ export class RoutineScheduler {
       // record it would have run is gone, and a firing needs one.
       return undefined;
     }
-    const record = decodeRoutineRecordV1(stored);
+    let record: RoutineRecordV1;
+    try {
+      record = decodeRoutineRecordV1(stored);
+    } catch {
+      // The waiting firing named a record nothing can read. Dropping the
+      // request is the same answer as a deleted record: a firing needs one.
+      return undefined;
+    }
     const fire: RoutineFireV1 = {
       schemaVersion: 1,
       routineId,
@@ -448,10 +596,17 @@ export class RoutineScheduler {
       record.timezone,
     );
     const anchor = new Date(record.updatedAt);
-    const late = now.getTime() - state.dueAt > ROUTINE_MISSED_GRACE_MS;
-    const missedCount = late
-      ? missedRoutineRunsV1(normalized, new Date(state.dueAt), now, anchor)
-      : 1;
+    // Every occurrence that has already elapsed coalesces into this one firing,
+    // not only the ones past the grace window. A `@every 1m` Routine after a
+    // four-minute stall used to run four Turns back to back, because inside
+    // the grace each occurrence was claimed on its own and one drain takes
+    // eight: lateness is lateness, and the rule is that it coalesces.
+    const missedCount = missedRoutineRunsV1(
+      normalized,
+      new Date(state.dueAt),
+      now,
+      anchor,
+    );
     const fire: RoutineFireV1 = {
       schemaVersion: 1,
       routineId: record.routineId,
@@ -468,10 +623,11 @@ export class RoutineScheduler {
       ...(missedCount > 1 ? { missedCount } : {}),
       entryId: `${routineFireIdV1(record.routineId, String(state.dueAt))}-entry`,
     };
-    // Recompute forward from now when the firing was late, and from the
-    // occurrence itself when it was on time; either way the clock advances
-    // before the Turn runs, so a crash mid-Turn cannot re-owe this occurrence.
-    const from = late ? now : new Date(state.dueAt);
+    // Recompute forward from now when the firing covered more than its own
+    // occurrence, and from the occurrence itself when it was on time; either
+    // way the clock advances before the Turn runs, so a crash mid-Turn cannot
+    // re-owe this occurrence.
+    const from = missedCount > 1 ? now : new Date(state.dueAt);
     const next = nextRoutineRunV1(normalized, from, anchor);
     await transaction.put(routineScheduleKeyV1(record.routineId), {
       schemaVersion: 1,
@@ -480,6 +636,9 @@ export class RoutineScheduler {
       dueAt: (
         next ?? new Date(now.getTime() + ROUTINE_MISSED_GRACE_MS)
       ).getTime(),
+      ...(state.consecutiveFailures === undefined
+        ? {}
+        : { consecutiveFailures: state.consecutiveFailures }),
     } satisfies RoutineScheduleStateV1);
     if (missedCount > 1) {
       // One entry says what was slept through. It is `skipped`, not `ok`: the
@@ -524,13 +683,150 @@ export class RoutineScheduler {
     } satisfies RoutineRecordV1);
   }
 
+  /**
+   * What a settled firing does to its Routine's clock.
+   *
+   * A firing that failed will most likely fail again on the next occurrence,
+   * and each attempt is a whole model Turn. Consecutive failures back the
+   * Routine off, and past the threshold it pauses itself and says why — where
+   * before it re-armed identically and burned a Turn a minute for ever with
+   * nothing on screen to say so.
+   */
+  async #recordOutcomeOnClock(
+    transaction: RoutineStorageWritesV1,
+    fire: RoutineFireV1,
+    outcome: RoutineFireOutcomeV1,
+    now: Date,
+  ): Promise<void> {
+    const stored = await transaction.get<unknown>(
+      routineScheduleKeyV1(fire.routineId),
+    );
+    if (stored === undefined) return;
+    let state: RoutineScheduleStateV1;
+    try {
+      state = decodeRoutineScheduleStateV1(stored);
+    } catch {
+      return;
+    }
+    if (outcome.status === "ok") {
+      if (state.consecutiveFailures === undefined) return;
+      const { consecutiveFailures: _cleared, ...cleared } = state;
+      await transaction.put(routineScheduleKeyV1(fire.routineId), cleared);
+      return;
+    }
+    const failures = (state.consecutiveFailures ?? 0) + 1;
+    if (failures >= ROUTINE_FAILURE_PAUSE_AFTER) {
+      const record = await transaction.get<unknown>(
+        routineKeyV1(fire.routineId),
+      );
+      if (record !== undefined) {
+        let decoded: RoutineRecordV1 | undefined;
+        try {
+          decoded = decodeRoutineRecordV1(record);
+        } catch {
+          decoded = undefined;
+        }
+        if (decoded && decoded.enabled) {
+          await transaction.put(routineKeyV1(fire.routineId), {
+            ...decoded,
+            enabled: false,
+            updatedAt: now.toISOString(),
+          } satisfies RoutineRecordV1);
+          await appendRoutineRunEntryV1(transaction, {
+            schemaVersion: 1,
+            entryId: `${fire.entryId}-paused`,
+            routineId: fire.routineId,
+            runId: fire.fireId,
+            fireId: fire.fireId,
+            trigger: fire.trigger,
+            status: "skipped",
+            startedAt: now.toISOString(),
+            finishedAt: now.toISOString(),
+            summary: `This Routine failed ${failures} times in a row and has been turned off. Turn it back on once the cause is fixed.`,
+          } satisfies RoutineRunEntryV1);
+        }
+      }
+    }
+    const backoff =
+      ROUTINE_FAILURE_BACKOFF_MS[
+        Math.min(failures, ROUTINE_FAILURE_BACKOFF_MS.length) - 1
+      ] ?? 0;
+    await transaction.put(routineScheduleKeyV1(fire.routineId), {
+      ...state,
+      consecutiveFailures: failures,
+      deferredUntil: Math.max(
+        state.deferredUntil ?? 0,
+        now.getTime() + backoff,
+      ),
+    } satisfies RoutineScheduleStateV1);
+  }
+
+  /**
+   * The name a person knows this Routine by, for anything addressed to them.
+   * Falls back to the id when the record is gone or unreadable — a message
+   * about a Routine is worth more than nothing, even under a broken record.
+   */
+  async #routineName(
+    transaction: RoutineStorageWritesV1,
+    routineId: string,
+  ): Promise<string> {
+    const stored = await transaction.get<unknown>(routineKeyV1(routineId));
+    if (stored === undefined) return routineId;
+    try {
+      return decodeRoutineRecordV1(stored).name;
+    } catch {
+      return routineId;
+    }
+  }
+
+  /**
+   * What a person is told about a firing that did not work.
+   *
+   * A completed firing has always written a completion-inbox entry; a failed
+   * one wrote nothing at all — no entry, no badge, no notification — so six
+   * consecutive failures left the header count untouched and the only trace
+   * was behind Bot settings → Routines → Run log → expand a row. "A Routine
+   * that stops working tells you." The entry is written in the transaction
+   * that settles the firing, so there is no window where a firing has failed
+   * and its outcome is nowhere, and it is idempotent on the run id.
+   */
+  async #recordFailureInbox(
+    transaction: RoutineStorageWritesV1,
+    fire: RoutineFireV1,
+    outcome: RoutineFireOutcomeV1,
+    now: string,
+  ): Promise<void> {
+    const name = await this.#routineName(transaction, fire.routineId);
+    const verb = outcome.status === "cancelled" ? "was stopped" : "did not run";
+    const records = await routineTerminalRecordsV1({
+      runId: fire.fireId,
+      routineId: fire.routineId,
+      routineName: name,
+      responseText:
+        outcome.summary === undefined || outcome.summary.trim().length === 0
+          ? `"${name}" ${verb}.`
+          : `"${name}" ${verb}: ${outcome.summary}`,
+      now,
+      read: (key) => transaction.get(key),
+    });
+    if (!records) return;
+    for (const [key, value] of Object.entries(records.records)) {
+      await transaction.put(key, value);
+    }
+  }
+
   async #settleFiring(
     fire: RoutineFireV1,
     outcome: RoutineFireOutcomeV1,
   ): Promise<void> {
-    const finishedAt = this.#now().toISOString();
+    const at = this.#now();
+    const finishedAt = at.toISOString();
     await this.#storage.transaction(async (transaction) => {
       await transaction.delete(routineFireKeyV1(fire.routineId));
+      await this.#recordOutcomeOnClock(transaction, fire, outcome, at);
+      if (outcome.status !== "ok") {
+        await this.#recordFailureInbox(transaction, fire, outcome, finishedAt);
+      }
       const startedAt = fire.mintedAt;
       await appendRoutineRunEntryV1(transaction, {
         schemaVersion: 1,

package/src/storage-keys.ts

@@ -46,6 +46,50 @@ export const ROUTINE_MISSED_GRACE_MS = 5 * 60_000;
 /** How long a deferral holds the alarm off a Routine while the object is busy. */
 export const ROUTINE_DEFERRAL_MS = 15_000;
 
+/**
+ * How long a Routine is held off after each consecutive failed firing.
+ *
+ * A firing that failed will most likely fail again on the next occurrence, and
+ * an occurrence is a whole model Turn. The last entry repeats for every failure
+ * past it, until the auto-pause.
+ */
+export const ROUTINE_FAILURE_BACKOFF_MS = [
+  30_000,
+  2 * 60_000,
+  10 * 60_000,
+  30 * 60_000,
+] as const;
+
+/**
+ * How many consecutive failures before a Routine pauses itself.
+ *
+ * A Routine that has failed five times running is broken, not unlucky, and
+ * burning a Turn an occurrence for ever is unbounded model spend nobody asked
+ * for. Pausing is durable, visible in the panel, and reversible by the person
+ * who owns it.
+ */
+export const ROUTINE_FAILURE_PAUSE_AFTER = 5;
+
+/**
+ * How long one firing may hold its Routine's lock before the alarm settles it
+ * for dead.
+ *
+ * The lock is `routine-fire:<id>`, written before the Turn and deleted when it
+ * settles. An isolate killed mid-firing leaves it behind, and without a lease
+ * that Routine is skipped for ever — and, because a locked Routine used to
+ * contribute no deadline, the object's alarm was deleted with it. The lease is
+ * what makes an abandoned firing a deadline rather than a silence.
+ */
+export const ROUTINE_FIRE_LEASE_MS = 10 * 60_000;
+
+/**
+ * How long one firing's Turn may run before the scheduler stops waiting on it.
+ *
+ * Strictly inside the lease, so the firing that timed out settles itself rather
+ * than waiting to be reaped.
+ */
+export const ROUTINE_FIRE_TIMEOUT_MS = 5 * 60_000;
+
 /**
  * Run entries are keyed by a descending sequence so a prefix listing returns
  * the newest first without reading the whole log.