@frockbot/kernel-do 0.0.0 → 0.1.1

package/src/composition-failures.test.ts

@@ -0,0 +1,405 @@
+import { describe, expect, test } from "bun:test";
+import {
+  activateCompositionV1,
+  CompositionMountFailureError,
+  type CompositionFailureV1,
+} from "@frockbot/kernel-composition/activation";
+import {
+  bootstrapGeneration,
+  compositionArtifactSetHashV1,
+  compositionGenerationIdV1,
+  type CompositionGenerationV1,
+  type CompositionMemberV1,
+  type MountedComposition,
+} from "@frockbot/kernel-composition/generation";
+import type { Context } from "cordis";
+import { DurableCompositionFailureLog } from "./composition-failures.ts";
+import { DurableCompositionStore } from "./composition-store.ts";
+import {
+  COMPOSITION_CURRENT_KEY,
+  compositionFailureCountKey,
+  compositionFailureKey,
+  compositionQuarantineKey,
+} from "./storage-keys.ts";
+
+class MemoryStorage {
+  readonly values = new Map<string, unknown>();
+
+  get<T>(key: string): Promise<T | undefined> {
+    return Promise.resolve(this.values.get(key) as T | undefined);
+  }
+
+  put(key: string | Record<string, unknown>, value?: unknown): Promise<void> {
+    if (typeof key === "string") this.values.set(key, structuredClone(value));
+    else {
+      for (const [entry, item] of Object.entries(key)) {
+        this.values.set(entry, structuredClone(item));
+      }
+    }
+    return Promise.resolve();
+  }
+
+  delete(key: string): Promise<boolean> {
+    return Promise.resolve(this.values.delete(key));
+  }
+
+  list<T>(options: { prefix?: string }): Promise<Map<string, T>> {
+    return Promise.resolve(
+      new Map(
+        [...this.values.entries()]
+          .filter(([key]) => key.startsWith(options.prefix ?? ""))
+          .sort(([left], [right]) => left.localeCompare(right)) as Array<
+          [string, T]
+        >,
+      ),
+    );
+  }
+
+  transaction<T>(callback: (storage: MemoryStorage) => Promise<T>): Promise<T> {
+    return callback(this);
+  }
+}
+
+const authoredMember: CompositionMemberV1 = {
+  packageId: "bot-authored-greeter",
+  specifier: "bot:greeter",
+  version: "0.0.1",
+  manifestHash: "a".repeat(64),
+  provenance: {
+    kind: "bot",
+    packageId: "bot-authored-greeter",
+    version: "0.0.1",
+    botId: "primary",
+    sessionId: "user-1:primary",
+    turnId: "turn-1",
+    runId: "run-1",
+    authoredAt: "2026-08-31T12:00:00.000Z",
+  },
+  artifact: {
+    contentHash: "b".repeat(64),
+    size: 512,
+    mediaType: "application/javascript",
+    bundlerVersion: "worker-bundler@0.2.3",
+  },
+};
+
+function bootstrap(): Promise<CompositionGenerationV1> {
+  return bootstrapGeneration(
+    [
+      {
+        packageId: "shell",
+        specifier: "@frockbot/plugin-shell",
+        version: "0.0.1",
+        manifest: { id: "shell", version: "0.0.1" },
+      },
+    ],
+    { createdAt: "2026-08-31T00:00:00.000Z" },
+  );
+}
+
+async function authored(
+  parent: CompositionGenerationV1,
+  createdAt: string,
+): Promise<CompositionGenerationV1> {
+  const members = [...parent.members, authoredMember].sort((left, right) =>
+    left.packageId.localeCompare(right.packageId),
+  );
+  const artifactSetHash = await compositionArtifactSetHashV1(members);
+  return {
+    schemaVersion: 1,
+    generationId: compositionGenerationIdV1(createdAt, artifactSetHash),
+    artifactSetHash,
+    parentGenerationId: parent.generationId,
+    createdAt,
+    origin: {
+      kind: "bot-authored",
+      runId: "run-1",
+      sessionId: "user-1:primary",
+      turnId: "turn-1",
+    },
+    members,
+    status: "pending",
+  };
+}
+
+function mounted(generation: CompositionGenerationV1): MountedComposition {
+  return {
+    generation,
+    root: {} as Context,
+    verify: () => Promise.resolve(),
+    dispose: () => Promise.resolve(),
+  };
+}
+
+interface Fixture {
+  storage: MemoryStorage;
+  store: DurableCompositionStore;
+  failures: DurableCompositionFailureLog;
+  lastKnownGood: CompositionGenerationV1;
+  broken: CompositionGenerationV1;
+}
+
+async function fixture(): Promise<Fixture> {
+  const storage = new MemoryStorage();
+  const state = { storage } as unknown as DurableObjectState;
+  const store = new DurableCompositionStore({ state, bootstrap });
+  const failures = new DurableCompositionFailureLog({
+    state,
+    now: () => new Date("2026-09-01T00:00:00.000Z"),
+  });
+  const lastKnownGood = await store.current();
+  const broken = await authored(lastKnownGood, "2026-09-01T00:00:00.000Z");
+  await store.propose(broken, { pin: true });
+  return { storage, store, failures, lastKnownGood, broken };
+}
+
+function activationStore(store: DurableCompositionStore) {
+  return {
+    read: (generationId: string) => store.read(generationId),
+    lastKnownGood: () => store.lastKnownGood(),
+    commit: (generationId: string) => store.commit(generationId),
+    fail: (generationId: string, options: { quarantined: boolean }) =>
+      store.fail(generationId, options),
+  };
+}
+
+/** Mounts everything except the named generation, which fails at `phase`. */
+function brokenHost(
+  brokenGenerationId: string,
+  phase: "resolve" | "mount" | "health",
+) {
+  return {
+    mount: (generation: CompositionGenerationV1) =>
+      generation.generationId === brokenGenerationId
+        ? Promise.reject(
+            new CompositionMountFailureError(
+              phase,
+              `package "bot-authored-greeter" failed at ${phase}`,
+              [`${phase}: diagnostic`],
+            ),
+          )
+        : Promise.resolve(mounted(generation)),
+  };
+}
+
+describe("fail-closed Composition activation", () => {
+  test("a broken generation leaves the last known good running and records a visible failure", async () => {
+    const { store, failures, storage, lastKnownGood, broken } = await fixture();
+    const raised: CompositionFailureV1[] = [];
+
+    const activation = await activateCompositionV1({
+      generationId: broken.generationId,
+      store: activationStore(store),
+      failures,
+      host: brokenHost(broken.generationId, "mount"),
+      signal: new AbortController().signal,
+      now: () => new Date("2026-09-01T00:01:00.000Z"),
+      onFailure: (failure) => {
+        raised.push(failure);
+        return Promise.resolve();
+      },
+    });
+
+    expect(activation.status).toBe("failed-closed");
+    if (activation.status !== "failed-closed") return;
+    expect(activation.mounted.generation.generationId).toBe(
+      lastKnownGood.generationId,
+    );
+    expect(activation.quarantined).toBe(false);
+    expect(activation.failure).toMatchObject({
+      generationId: broken.generationId,
+      attempt: 1,
+      phase: "mount",
+    });
+    // The failure is durable, visible, and repairable.
+    expect(raised).toHaveLength(1);
+    expect(
+      storage.values.get(compositionFailureKey(broken.generationId, 1)),
+    ).toMatchObject({ attempt: 1, phase: "mount" });
+    expect(
+      storage.values.get(compositionFailureCountKey(broken.generationId)),
+    ).toBe(1);
+    expect((await store.read(broken.generationId))?.status).toBe("failed");
+    expect((await store.lastKnownGood()).generationId).toBe(
+      lastKnownGood.generationId,
+    );
+  });
+
+  test("each of the three load sites records its own phase", async () => {
+    for (const phase of ["resolve", "mount", "health"] as const) {
+      const { store, failures, broken } = await fixture();
+      const activation = await activateCompositionV1({
+        generationId: broken.generationId,
+        store: activationStore(store),
+        failures,
+        host: brokenHost(broken.generationId, phase),
+        signal: new AbortController().signal,
+      });
+      expect(activation.status).toBe("failed-closed");
+      if (activation.status !== "failed-closed") return;
+      expect(activation.failure?.phase).toBe(phase);
+      expect(activation.failure?.diagnostics).toEqual([`${phase}: diagnostic`]);
+    }
+  });
+
+  test("an unresolvable generation fails closed at the resolve phase", async () => {
+    const { store, failures, lastKnownGood } = await fixture();
+    const activation = await activateCompositionV1({
+      generationId: "2026-09-02T00:00:00.000Z:deadbeefdeadbeef",
+      store: activationStore(store),
+      failures,
+      host: { mount: (generation) => Promise.resolve(mounted(generation)) },
+      signal: new AbortController().signal,
+    });
+    expect(activation.status).toBe("failed-closed");
+    if (activation.status !== "failed-closed") return;
+    expect(activation.failure?.phase).toBe("resolve");
+    expect(activation.mounted.generation.generationId).toBe(
+      lastKnownGood.generationId,
+    );
+  });
+
+  test("a third consecutive failure quarantines the generation and is never retried", async () => {
+    const { store, failures, storage, lastKnownGood, broken } = await fixture();
+    const host = brokenHost(broken.generationId, "health");
+    const activationStoreForTurn = activationStore(store);
+
+    for (const attempt of [1, 2, 3]) {
+      // Every Turn re-pins the failed generation until it is quarantined.
+      if (attempt > 1) {
+        await storage.put(COMPOSITION_CURRENT_KEY, {
+          generationId: broken.generationId,
+          artifactSetHash: broken.artifactSetHash,
+        });
+      }
+      const activation = await activateCompositionV1({
+        generationId: broken.generationId,
+        store: activationStoreForTurn,
+        failures,
+        host,
+        signal: new AbortController().signal,
+      });
+      expect(activation.status).toBe("failed-closed");
+      if (activation.status !== "failed-closed") return;
+      expect(activation.failure?.attempt).toBe(attempt);
+      expect(activation.quarantined).toBe(attempt === 3);
+    }
+
+    expect((await store.read(broken.generationId))?.status).toBe("quarantined");
+    expect(
+      storage.values.get(compositionQuarantineKey(broken.generationId)),
+    ).toMatchObject({ failures: 3 });
+    // Quarantine moved the pointer back, so the fourth Turn pins the good one.
+    expect((await store.current()).generationId).toBe(
+      lastKnownGood.generationId,
+    );
+
+    // A fourth activation of the quarantined generation attempts no mount.
+    let attempted = 0;
+    const fourth = await activateCompositionV1({
+      generationId: broken.generationId,
+      store: activationStoreForTurn,
+      failures,
+      host: {
+        mount: (generation) => {
+          if (generation.generationId === broken.generationId) attempted += 1;
+          return Promise.resolve(mounted(generation));
+        },
+      },
+      signal: new AbortController().signal,
+    });
+    expect(attempted).toBe(0);
+    expect(fourth.status).toBe("failed-closed");
+    if (fourth.status !== "failed-closed") return;
+    expect(fourth.quarantined).toBe(true);
+    expect(await failures.list(broken.generationId)).toHaveLength(3);
+  });
+
+  test("quarantine is per generation, so a later unrelated generation still activates", async () => {
+    const { store, failures, broken, lastKnownGood } = await fixture();
+    await activateCompositionV1({
+      generationId: broken.generationId,
+      store: activationStore(store),
+      failures,
+      host: brokenHost(broken.generationId, "mount"),
+      signal: new AbortController().signal,
+    });
+    await failures.record({
+      generationId: broken.generationId,
+      at: "2026-09-01T00:02:00.000Z",
+      phase: "mount",
+      message: "second",
+      diagnostics: [],
+    });
+    await failures.record({
+      generationId: broken.generationId,
+      at: "2026-09-01T00:03:00.000Z",
+      phase: "mount",
+      message: "third",
+      diagnostics: [],
+    });
+    await store.fail(broken.generationId, { quarantined: true });
+
+    const later = await authored(lastKnownGood, "2026-09-03T00:00:00.000Z");
+    await store.propose(later, { pin: true });
+    const activation = await activateCompositionV1({
+      generationId: later.generationId,
+      store: activationStore(store),
+      failures,
+      host: { mount: (generation) => Promise.resolve(mounted(generation)) },
+      signal: new AbortController().signal,
+    });
+
+    expect(activation.status).toBe("activated");
+    expect((await store.read(later.generationId))?.status).toBe("active");
+    expect((await store.lastKnownGood()).generationId).toBe(later.generationId);
+    expect(await failures.quarantine(later.generationId)).toBeUndefined();
+    expect(await failures.quarantine(broken.generationId)).toMatchObject({
+      failures: 3,
+    });
+    expect((await store.read(broken.generationId))?.status).toBe("quarantined");
+  });
+
+  test("a generation that finally activates commits and clears its consecutive count", async () => {
+    const { store, failures, storage, broken } = await fixture();
+    await activateCompositionV1({
+      generationId: broken.generationId,
+      store: activationStore(store),
+      failures,
+      host: brokenHost(broken.generationId, "mount"),
+      signal: new AbortController().signal,
+    });
+    expect(
+      storage.values.get(compositionFailureCountKey(broken.generationId)),
+    ).toBe(1);
+
+    const activation = await activateCompositionV1({
+      generationId: broken.generationId,
+      store: activationStore(store),
+      failures,
+      host: { mount: (generation) => Promise.resolve(mounted(generation)) },
+      signal: new AbortController().signal,
+    });
+
+    expect(activation.status).toBe("activated");
+    expect((await store.read(broken.generationId))?.status).toBe("active");
+    expect(
+      storage.values.get(compositionFailureCountKey(broken.generationId)),
+    ).toBeUndefined();
+    // The recorded failure survives: it is the repair history a User reads.
+    expect(await failures.list(broken.generationId)).toHaveLength(1);
+  });
+
+  test("a last known good that will not mount has nothing to fail into", async () => {
+    const { store, failures, lastKnownGood } = await fixture();
+    await expect(
+      activateCompositionV1({
+        generationId: lastKnownGood.generationId,
+        store: activationStore(store),
+        failures,
+        host: brokenHost(lastKnownGood.generationId, "mount"),
+        signal: new AbortController().signal,
+      }),
+    ).rejects.toThrow(/failed at mount/);
+  });
+});

package/src/composition-failures.ts

@@ -0,0 +1,108 @@
+/// <reference types="@cloudflare/workers-types" />
+// The Bot Durable Object is the authority for why a Composition generation
+// failed to activate and for whether it is quarantined. Failures are durable,
+// visible, and repairable: nothing here deletes a recorded failure, and the
+// consecutive counter is reset only by a generation that finally activates.
+import {
+  COMPOSITION_QUARANTINE_THRESHOLD,
+  decodeCompositionFailureV1,
+  decodeCompositionQuarantineV1,
+  type CompositionFailureInputV1,
+  type CompositionFailureLog,
+  type CompositionFailureOutcomeV1,
+  type CompositionFailureV1,
+  type CompositionQuarantineV1,
+} from "@frockbot/kernel-composition/activation";
+import {
+  compositionFailureCountKey,
+  compositionFailureKey,
+  compositionFailurePrefix,
+  compositionQuarantineKey,
+} from "./storage-keys.js";
+
+export interface DurableCompositionFailureLogOptions {
+  state: DurableObjectState;
+  /** Injected clock; the quarantine record is stamped with it. */
+  now?(): Date;
+  /** Consecutive failures that quarantine a generation. Defaults to three. */
+  threshold?: number;
+}
+
+/**
+ * `CompositionFailureLog` over the Bot object's prefixed keys:
+ * `composition:failure:<generationId>:<attempt>`,
+ * `composition:failure-count:<generationId>`, and
+ * `composition:quarantine:<generationId>`.
+ *
+ * Quarantine is per generation, never per Bot: a later, unrelated generation is
+ * unaffected by an earlier one being quarantined.
+ */
+export class DurableCompositionFailureLog implements CompositionFailureLog {
+  private readonly ctx: DurableObjectState;
+  private readonly now: () => Date;
+  private readonly threshold: number;
+
+  constructor(options: DurableCompositionFailureLogOptions) {
+    this.ctx = options.state;
+    this.now = options.now ?? (() => new Date());
+    this.threshold = options.threshold ?? COMPOSITION_QUARANTINE_THRESHOLD;
+  }
+
+  /** The attempt number is assigned here, inside the counter's transaction. */
+  async record(
+    failure: CompositionFailureInputV1,
+  ): Promise<CompositionFailureOutcomeV1> {
+    const generationId = failure.generationId;
+    return this.ctx.storage.transaction(async (transaction) => {
+      const previous =
+        (await transaction.get<number>(
+          compositionFailureCountKey(generationId),
+        )) ?? 0;
+      const attempt = previous + 1;
+      const recorded = decodeCompositionFailureV1({ ...failure, attempt });
+      const quarantined = attempt >= this.threshold;
+      const writes: Record<string, unknown> = {
+        [compositionFailureKey(generationId, attempt)]: recorded,
+        [compositionFailureCountKey(generationId)]: attempt,
+      };
+      if (quarantined) {
+        writes[compositionQuarantineKey(generationId)] =
+          decodeCompositionQuarantineV1({
+            generationId,
+            quarantinedAt: this.now().toISOString(),
+            reason: recorded.message,
+            failures: attempt,
+          });
+      }
+      await transaction.put(writes);
+      return { consecutiveFailures: attempt, quarantined };
+    });
+  }
+
+  /** Oldest attempt first. */
+  async list(generationId: string): Promise<CompositionFailureV1[]> {
+    const entries = await this.ctx.storage.list<unknown>({
+      prefix: compositionFailurePrefix(generationId),
+    });
+    return [...entries.values()].map(decodeCompositionFailureV1);
+  }
+
+  async quarantine(
+    generationId: string,
+  ): Promise<CompositionQuarantineV1 | undefined> {
+    const stored = await this.ctx.storage.get<unknown>(
+      compositionQuarantineKey(generationId),
+    );
+    return stored === undefined
+      ? undefined
+      : decodeCompositionQuarantineV1(stored);
+  }
+
+  /**
+   * A generation that finally activates starts its consecutive count over. The
+   * recorded failures survive: they are the repair history a User reads.
+   */
+  async clear(generationId: string): Promise<void> {
+    await this.ctx.storage.delete(compositionFailureCountKey(generationId));
+  }
+}