@frockbot/kernel-do 0.3.9 → 0.3.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/kernel-do",
-  "version": "0.3.9",
+  "version": "0.3.11",
   "private": false,
   "type": "module",
   "exports": {
@@ -12,8 +12,8 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-composition": "0.3.9",
-    "@frockbot/kernel-contracts": "0.3.9",
+    "@frockbot/kernel-composition": "0.3.11",
+    "@frockbot/kernel-contracts": "0.3.11",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/composition-failures.test.ts

@@ -39,7 +39,15 @@ class MemoryStorage {
     return Promise.resolve();
   }
 
-  delete(key: string): Promise<boolean> {
+  delete(key: string | string[]): Promise<boolean | number> {
+    if (Array.isArray(key)) {
+      return Promise.resolve(
+        key.reduce(
+          (count, entry) => count + (this.values.delete(entry) ? 1 : 0),
+          0,
+        ),
+      );
+    }
     return Promise.resolve(this.values.delete(key));
   }
 
@@ -390,6 +398,133 @@ describe("fail-closed Composition activation", () => {
     expect(await failures.list(broken.generationId)).toHaveLength(1);
   });
 
+  test("three failed repairs quarantine, even though each one is a new generation", async () => {
+    // The safeguard was dead code in the path a real user takes. The model
+    // never retries a failed generation: it authors a *new* one, which
+    // supersedes the failed one at attempt 1, so the per-generation counter
+    // never reached three and the Composition just grew one dead generation
+    // per repair attempt.
+    const { store, failures, storage, lastKnownGood } = await fixture();
+    const attempts: CompositionGenerationV1[] = [];
+    for (const minute of [1, 2, 3]) {
+      const generation = await authored(
+        lastKnownGood,
+        `2026-09-01T00:0${minute}:00.000Z`,
+      );
+      await store.propose(generation, { pin: true });
+      attempts.push(generation);
+      const activation = await activateCompositionV1({
+        generationId: generation.generationId,
+        store: activationStore(store),
+        failures,
+        host: brokenHost(generation.generationId, "mount"),
+        signal: new AbortController().signal,
+      });
+      expect(activation.status).toBe("failed-closed");
+      if (activation.status !== "failed-closed") return;
+      // Each new generation is on its own first attempt...
+      expect(activation.failure?.attempt).toBe(1);
+      // ...but the Bot's streak is what earns the quarantine.
+      expect(activation.quarantined).toBe(minute === 3);
+    }
+
+    const third = attempts[2]!;
+    expect((await store.read(third.generationId))?.status).toBe("quarantined");
+    expect(
+      storage.values.get(compositionQuarantineKey(third.generationId)),
+    ).toBeDefined();
+    // The two earlier attempts stay `failed`: a quarantine is a decision about
+    // the generation that earned it, not a verdict on its history.
+    expect((await store.read(attempts[0]!.generationId))?.status).toBe(
+      "failed",
+    );
+    expect((await store.read(attempts[1]!.generationId))?.status).toBe(
+      "failed",
+    );
+  });
+
+  test("a generation that activates clears the streak, so the next failure starts over", async () => {
+    const { store, failures, lastKnownGood } = await fixture();
+    for (const minute of [1, 2]) {
+      const generation = await authored(
+        lastKnownGood,
+        `2026-09-01T00:0${minute}:00.000Z`,
+      );
+      await store.propose(generation, { pin: true });
+      await activateCompositionV1({
+        generationId: generation.generationId,
+        store: activationStore(store),
+        failures,
+        host: brokenHost(generation.generationId, "mount"),
+        signal: new AbortController().signal,
+      });
+    }
+
+    const working = await authored(lastKnownGood, "2026-09-01T00:03:00.000Z");
+    await store.propose(working, { pin: true });
+    const activated = await activateCompositionV1({
+      generationId: working.generationId,
+      store: activationStore(store),
+      failures,
+      host: { mount: (generation) => Promise.resolve(mounted(generation)) },
+      signal: new AbortController().signal,
+    });
+    expect(activated.status).toBe("activated");
+
+    const next = await authored(lastKnownGood, "2026-09-01T00:04:00.000Z");
+    await store.propose(next, { pin: true });
+    const activation = await activateCompositionV1({
+      generationId: next.generationId,
+      store: activationStore(store),
+      failures,
+      host: brokenHost(next.generationId, "mount"),
+      signal: new AbortController().signal,
+    });
+
+    expect(activation.status).toBe("failed-closed");
+    if (activation.status !== "failed-closed") return;
+    expect(activation.quarantined).toBe(false);
+  });
+
+  test("the streak expires, so unrelated failures far apart are separate incidents", async () => {
+    // Three repair attempts are one sitting. Three failures months apart are
+    // three incidents, and quarantining the third would strand a Bot for
+    // something it had already recovered from twice.
+    const storage = new MemoryStorage();
+    const state = { storage } as unknown as DurableObjectState;
+    const store = new DurableCompositionStore({ state, bootstrap });
+    let clock = new Date("2026-09-01T00:00:00.000Z");
+    const failures = new DurableCompositionFailureLog({
+      state,
+      now: () => clock,
+      streakWindowMs: 60 * 60 * 1000,
+    });
+    const lastKnownGood = await store.current();
+
+    const outcomes: boolean[] = [];
+    for (const day of [1, 2, 3]) {
+      clock = new Date(`2026-09-0${day}T00:00:00.000Z`);
+      const generation = await authored(
+        lastKnownGood,
+        `2026-09-0${day}T00:00:00.000Z`,
+      );
+      await store.propose(generation, { pin: true });
+      const activation = await activateCompositionV1({
+        generationId: generation.generationId,
+        store: activationStore(store),
+        failures,
+        host: brokenHost(generation.generationId, "mount"),
+        signal: new AbortController().signal,
+        now: () => clock,
+      });
+      if (activation.status !== "failed-closed") throw new Error("expected");
+      outcomes.push(activation.quarantined);
+    }
+
+    // A day apart each time: never a streak, so never a quarantine.
+    expect(outcomes).toEqual([false, false, false]);
+  });
+
   test("a last known good that will not mount has nothing to fail into", async () => {
     const { store, failures, lastKnownGood } = await fixture();
     await expect(

package/src/composition-failures.ts

@@ -14,6 +14,7 @@ import {
   type CompositionQuarantineV1,
 } from "@frockbot/kernel-composition/activation";
 import {
+  COMPOSITION_FAILURE_STREAK_KEY,
   compositionFailureCountKey,
   compositionFailureKey,
   compositionFailurePrefix,
@@ -26,26 +27,47 @@ export interface DurableCompositionFailureLogOptions {
   now?(): Date;
   /** Consecutive failures that quarantine a generation. Defaults to three. */
   threshold?: number;
+  /** How long the Bot-wide streak stays alive. Defaults to one hour. */
+  streakWindowMs?: number;
 }
 
+/** The Bot's live failure streak: how many, and when the last one landed. */
+interface CompositionFailureStreakV1 {
+  count: number;
+  at: string;
+}
+
+/**
+ * One sitting. Repair attempts arrive minutes apart, so an hour is long enough
+ * to span a Bot's whole attempt to fix itself and short enough that yesterday's
+ * unrelated failure does not count towards today's quarantine.
+ */
+const COMPOSITION_FAILURE_STREAK_WINDOW_MS = 60 * 60 * 1000;
+
 /**
  * `CompositionFailureLog` over the Bot object's prefixed keys:
  * `composition:failure:<generationId>:<attempt>`,
  * `composition:failure-count:<generationId>`, and
  * `composition:quarantine:<generationId>`.
  *
- * Quarantine is per generation, never per Bot: a later, unrelated generation is
- * unaffected by an earlier one being quarantined.
+ * Quarantine is *marked* per generation, never per Bot: a generation an earlier
+ * one's quarantine does not implicate is unaffected, and lifting a quarantine
+ * is a decision about one generation. What earns a quarantine is either
+ * counter reaching the threshold — this generation's own retries, or the Bot's
+ * consecutive failures across however many generations the repairs minted.
  */
 export class DurableCompositionFailureLog implements CompositionFailureLog {
   private readonly ctx: DurableObjectState;
   private readonly now: () => Date;
   private readonly threshold: number;
+  private readonly streakWindowMs: number;
 
   constructor(options: DurableCompositionFailureLogOptions) {
     this.ctx = options.state;
     this.now = options.now ?? (() => new Date());
     this.threshold = options.threshold ?? COMPOSITION_QUARANTINE_THRESHOLD;
+    this.streakWindowMs =
+      options.streakWindowMs ?? COMPOSITION_FAILURE_STREAK_WINDOW_MS;
   }
 
   /** The attempt number is assigned here, inside the counter's transaction. */
@@ -60,10 +82,33 @@ export class DurableCompositionFailureLog implements CompositionFailureLog {
         )) ?? 0;
       const attempt = previous + 1;
       const recorded = decodeCompositionFailureV1({ ...failure, attempt });
-      const quarantined = attempt >= this.threshold;
+      // Two counters, one threshold. `attempt` is this generation's own
+      // retries; `streak` is the Bot's consecutive failures however many
+      // generations they are spread over. Only the second one ever moves when
+      // the model repairs by authoring a *new* generation, which is what it
+      // always does — so without it the safeguard never fired and the
+      // Composition grew one dead generation per repair attempt.
+      //
+      // The streak is bounded in time as well as reset by success. Three
+      // repair attempts are one sitting; three unrelated failures months apart
+      // are three separate incidents, and quarantining the third would strand
+      // a Bot for something it had already recovered from twice.
+      const at = this.now();
+      const stored = await transaction.get<CompositionFailureStreakV1>(
+        COMPOSITION_FAILURE_STREAK_KEY,
+      );
+      const continues =
+        stored !== undefined &&
+        at.getTime() - Date.parse(stored.at) <= this.streakWindowMs;
+      const streak = (continues ? stored.count : 0) + 1;
+      const quarantined = attempt >= this.threshold || streak >= this.threshold;
       const writes: Record<string, unknown> = {
         [compositionFailureKey(generationId, attempt)]: recorded,
         [compositionFailureCountKey(generationId)]: attempt,
+        [COMPOSITION_FAILURE_STREAK_KEY]: {
+          count: streak,
+          at: at.toISOString(),
+        } satisfies CompositionFailureStreakV1,
       };
       if (quarantined) {
         writes[compositionQuarantineKey(generationId)] =
@@ -71,7 +116,9 @@ export class DurableCompositionFailureLog implements CompositionFailureLog {
             generationId,
             quarantinedAt: this.now().toISOString(),
             reason: recorded.message,
-            failures: attempt,
+            // The count that earned it, which is the streak whenever the
+            // repairs were spread over generations.
+            failures: Math.max(attempt, streak),
           });
       }
       await transaction.put(writes);
@@ -103,6 +150,9 @@ export class DurableCompositionFailureLog implements CompositionFailureLog {
    * recorded failures survive: they are the repair history a User reads.
    */
   async clear(generationId: string): Promise<void> {
-    await this.ctx.storage.delete(compositionFailureCountKey(generationId));
+    await this.ctx.storage.delete([
+      compositionFailureCountKey(generationId),
+      COMPOSITION_FAILURE_STREAK_KEY,
+    ]);
   }
 }

package/src/storage-keys.ts

@@ -109,6 +109,19 @@ export function compositionFailureCountKey(generationId: string): string {
   return `${COMPOSITION_FAILURE_COUNT_PREFIX}${generationId}`;
 }
 
+/**
+ * The Bot's own consecutive-failure streak, across generations.
+ *
+ * The per-generation counter alone left quarantine as dead code in the path a
+ * real user takes: the model's natural repair is to *author a new generation*,
+ * which supersedes the failed one at attempt 1, so no generation ever reached
+ * three. Every repair attempt then added one more dead generation, forever.
+ * This key counts the Bot's consecutive activation failures however many
+ * generations they are spread over, and a generation that finally activates
+ * clears it.
+ */
+export const COMPOSITION_FAILURE_STREAK_KEY = "composition:failure-streak";
+
 export function compositionQuarantineKey(generationId: string): string {
   return `${COMPOSITION_QUARANTINE_PREFIX}${generationId}`;
 }