@frockbot/kernel-agent-loop 0.3.5 → 0.3.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/kernel-agent-loop",
-  "version": "0.3.5",
+  "version": "0.3.7",
   "private": false,
   "type": "module",
   "exports": {
@@ -12,13 +12,13 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-contracts": "0.3.5",
+    "@frockbot/kernel-contracts": "0.3.7",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {
-    "@frockbot/plugin-models": "0.3.5",
-    "@frockbot/plugin-prompt": "0.3.5",
-    "@frockbot/plugin-tools": "0.3.5",
+    "@frockbot/plugin-models": "0.3.7",
+    "@frockbot/plugin-prompt": "0.3.7",
+    "@frockbot/plugin-tools": "0.3.7",
     "@types/bun": "1.4.0",
     "@types/node": "26.2.0",
     "typescript": "^7.0.2"

package/src/deadlines.test.ts

@@ -0,0 +1,246 @@
+// Two Turns that never ended: one hung for seventeen minutes with nothing on
+// screen because nothing bounded a Turn's wall clock, and a provider that
+// rejected a request before it started took the whole Turn down rather than
+// being tried once more.
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  LlmEffectNotStartedError,
+  type LlmProvider,
+  SessionStore,
+} from "@frockbot/kernel-contracts";
+import { LlmRegistry } from "@frockbot/plugin-models";
+import { SystemPromptRegistry } from "@frockbot/plugin-prompt";
+import { ToolRegistry } from "@frockbot/plugin-tools";
+import { AgentRegistry } from "./agent.js";
+import { Context, type Plugin } from "cordis";
+import { AgentLoop, TURN_DEADLINE_REASON_V1 } from "./index.js";
+
+const roots: Context[] = [];
+const allowEffect = () => Promise.resolve(true);
+
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => root.fiber.dispose()));
+});
+
+async function mount(
+  provider: LlmProvider,
+  config: { turnDeadlineMs?: number } = {},
+): Promise<Context> {
+  const root = new Context();
+  roots.push(root);
+  await root.plugin(SessionStore, {});
+  await root.plugin(SystemPromptRegistry);
+  await root.plugin(LlmRegistry);
+  await root.plugin(ToolRegistry);
+  await root.plugin(AgentRegistry);
+  const promptPlugin: Plugin.Function = (ctx) =>
+    ctx.systemPrompt.register({ id: "identity", render: () => "Be useful." });
+  promptPlugin.inject = ["systemPrompt"];
+  const providerPlugin: Plugin.Function = (ctx) => ctx.llm.register(provider);
+  providerPlugin.inject = ["llm"];
+  await root.plugin(promptPlugin);
+  await root.plugin(providerPlugin);
+  await root.plugin(AgentLoop, {
+    maxSteps: 4,
+    composition: {
+      generationId: "generation-1",
+      artifactSetHash: "a".repeat(64),
+    },
+    ...config,
+  });
+  return root;
+}
+
+describe("a Turn that runs out of wall clock", () => {
+  test("ends as interrupted, saying why, instead of hanging", async () => {
+    const provider: LlmProvider = {
+      id: "silent",
+      // A provider that accepted the request and will never answer. This is
+      // the seventeen-minute Turn, reproduced.
+      // eslint-disable-next-line require-yield
+      async *stream(_request, signal) {
+        await new Promise((_resolve, reject) => {
+          signal.addEventListener("abort", () => reject(signal.reason), {
+            once: true,
+          });
+        });
+      },
+    };
+    const root = await mount(provider, { turnDeadlineMs: 25 });
+    const handle = await root.agents.create({
+      botId: "deadline-bot",
+      sessionId: "deadline",
+      provider: "silent",
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Take your time.");
+    await handle.agent.whenIdle();
+
+    // The model effect is unsettled, so the Turn is owed a reconciliation
+    // rather than a clean end — but it *ended*, which is the whole point, and
+    // ADR 0028 settles it from there.
+    const journal = handle.agent.session.events;
+    expect(
+      journal.some((event) => event.type === "model/reconciliation-required"),
+    ).toBe(true);
+    expect(handle.agent.status).toBe("idle");
+  });
+
+  test("is reported as the deadline, never as a Stop the person did not press", async () => {
+    const provider: LlmProvider = {
+      id: "never-reached",
+      // eslint-disable-next-line require-yield
+      async *stream() {
+        throw new Error("the Turn should never have got this far");
+      },
+    };
+    const root = await mount(provider, { turnDeadlineMs: 25 });
+    // Stalling before the first model request leaves no uncertain effect, so
+    // the Turn settles on its own terms and the reason it carries is the one
+    // under test. The deadline aborts the same controller Stop does; the
+    // person must not be told they stopped it.
+    let stalled: (() => void) | undefined;
+    root.on("agent/pre-step", async (_agent, _inputs, _turn, _step, next) => {
+      await new Promise<void>((_resolve, reject) => {
+        stalled = () => reject(new Error("stalled"));
+      });
+      return next();
+    });
+    // The deadline aborts the loop's controller, which the stalled hook does
+    // not itself watch; releasing it here stands in for whatever slow thing a
+    // real Turn was waiting on noticing that nobody is waiting any more.
+    setTimeout(() => stalled?.(), 60);
+    const handle = await root.agents.create({
+      botId: "deadline-reason-bot",
+      sessionId: "deadline-reason",
+      provider: "never-reached",
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Stall before the model.");
+    await handle.agent.whenIdle();
+
+    const end = handle.agent.session.events.findLast(
+      (event) => event.type === "turn/end",
+    );
+    if (end?.type !== "turn/end") throw new Error("the Turn never ended");
+    expect(end.outcome).toBe("interrupted");
+    expect(end.reason).toBe(TURN_DEADLINE_REASON_V1);
+  });
+});
+
+describe("a model request the provider says never started", () => {
+  test("is tried once more, and the Turn succeeds on the retry", async () => {
+    let attempts = 0;
+    const provider: LlmProvider = {
+      id: "flaky-binding",
+      async *stream() {
+        attempts += 1;
+        if (attempts === 1) {
+          throw new LlmEffectNotStartedError(
+            "model binding was not resolvable",
+          );
+        }
+        yield { type: "text-delta", text: "Second time lucky." } as const;
+        yield { type: "finish", reason: "completed" } as const;
+      },
+    };
+    const root = await mount(provider);
+    const handle = await root.agents.create({
+      botId: "retry-bot",
+      sessionId: "retry",
+      provider: "flaky-binding",
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Say hello.");
+    await handle.agent.whenIdle();
+
+    expect(attempts).toBe(2);
+    expect(handle.agent.session.events.at(-1)).toMatchObject({
+      type: "turn/end",
+      outcome: "completed",
+    });
+    // The retry is not a hidden event type: the durable log already shows the
+    // attempt that did not start and the one that replaced it.
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/request",
+      ),
+    ).toHaveLength(2);
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/effect-not-started",
+      ),
+    ).toHaveLength(1);
+  });
+
+  test("is not retried a second time", async () => {
+    let attempts = 0;
+    const provider: LlmProvider = {
+      id: "always-rejects",
+      async *stream() {
+        attempts += 1;
+        throw new LlmEffectNotStartedError("invalid api key");
+      },
+    };
+    const root = await mount(provider);
+    const handle = await root.agents.create({
+      botId: "no-retry-bot",
+      sessionId: "no-retry",
+      provider: "always-rejects",
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Say hello.");
+    await handle.agent.whenIdle();
+
+    expect(attempts).toBe(2);
+    expect(handle.agent.session.events.at(-1)).toMatchObject({
+      type: "turn/end",
+      outcome: "model-error",
+      reason: "invalid api key",
+    });
+  });
+
+  test("an uncertain failure is never retried", async () => {
+    let attempts = 0;
+    const provider: LlmProvider = {
+      id: "uncertain",
+      async *stream() {
+        attempts += 1;
+        // Not classified as unstarted: the call may well have run, so trying
+        // again would be a silent duplicate.
+        throw new Error("connection reset mid-stream");
+      },
+    };
+    const root = await mount(provider);
+    const handle = await root.agents.create({
+      botId: "uncertain-bot",
+      sessionId: "uncertain",
+      provider: "uncertain",
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Say hello.");
+    await handle.agent.whenIdle();
+
+    expect(attempts).toBe(1);
+    expect(
+      handle.agent.session.events.some(
+        (event) => event.type === "model/reconciliation-required",
+      ),
+    ).toBe(true);
+  });
+});
+
+// Named so a reader who greps for the reason string finds where it is set.
+test("the deadline reason tells the person what to do about it", () => {
+  expect(TURN_DEADLINE_REASON_V1).toContain("Try sending it again");
+});

package/src/index.test.ts

@@ -1223,7 +1223,7 @@ describe("AgentLoop", () => {
     ).toBe(false);
   });
 
-  test("keeps an unretrievable provider effect open without repeating it", async () => {
+  test("settles a turn whose provider cannot retrieve the effect", async () => {
     let streams = 0;
     const provider: LlmProvider = {
       id: "unretrievable",
@@ -1279,28 +1279,26 @@ describe("AgentLoop", () => {
       model: "model-1",
     });
 
-    handle.agent.resume();
-    await handle.agent.whenIdle();
     handle.agent.resume();
     await handle.agent.whenIdle();
 
+    // A provider that declares no retrieval will never grow one, so the run
+    // settles as a model error rather than parking on a reconciliation that
+    // can never happen (and throwing out of the Durable Object).
     expect(streams).toBe(0);
-    expect(
-      handle.agent.session.events.filter(
-        (event) => event.type === "model/reconciliation-required",
-      ),
-    ).toEqual([
-      expect.objectContaining({
-        requestId: "unretrievable-effect-1",
-        reason:
-          'LLM provider "unretrievable" does not support provider-bound retrieval',
-      }),
-    ]);
     expect(
       handle.agent.session.events.some(
-        (event) => event.type === "step/end" || event.type === "turn/end",
+        (event) => event.type === "model/reconciliation-required",
       ),
     ).toBe(false);
+    const turnEnd = handle.agent.session.events.findLast(
+      (event) => event.type === "turn/end",
+    );
+    expect(turnEnd).toMatchObject({
+      outcome: "model-error",
+      reason:
+        'LLM provider "unretrievable" does not support provider-bound retrieval',
+    });
   });
 
   test("keeps an ambiguous dispatched effect open for provider reconciliation", async () => {

package/src/index.ts

@@ -41,6 +41,11 @@ declare module "cordis" {
 
 export interface AgentLoopConfig {
   maxSteps?: number;
+  /**
+   * The wall clock one Turn is allowed, in milliseconds. Defaults to
+   * {@link TURN_DEADLINE_MS_V1}; named by a caller only to test it.
+   */
+  turnDeadlineMs?: number;
   /** The Composition generation this mounted root was pinned to at admission. */
   composition: CompositionPinV1;
 }
@@ -66,7 +71,8 @@ interface ModelResponse {
 
 type ModelReconciliation =
   | { status: "recovered"; response: ModelResponse }
-  | { status: "unavailable"; reason: string };
+  | { status: "unavailable"; reason: string }
+  | { status: "not-retrievable"; reason: string };
 
 class ModelEffectReconciliationRequiredError extends Error {
   constructor(
@@ -103,6 +109,29 @@ class StepLimitReachedError extends Error {
   }
 }
 
+/**
+ * The longest a single Turn may run before the loop stops waiting for it.
+ *
+ * Nothing bounded a Turn's wall clock before this: one hung for seventeen
+ * minutes with an animated avatar and nothing else, and would have hung until
+ * the isolate died. Fifteen minutes is well past any Turn a person is watching
+ * and well inside the point at which they have concluded the product is broken.
+ */
+export const TURN_DEADLINE_MS_V1 = 15 * 60 * 1000;
+
+/**
+ * How many times one step will send its model request.
+ *
+ * Two: the first attempt and one retry. It applies only to a failure the
+ * provider classified as never having started, so no retry can duplicate a
+ * call that may already have run.
+ */
+export const MODEL_REQUEST_ATTEMPTS_V1 = 2;
+
+/** What a `turn/end` records when the Turn ran out of wall clock. */
+export const TURN_DEADLINE_REASON_V1 =
+  "This Turn ran for 15 minutes without finishing and was stopped. Try sending it again.";
+
 /** Durable Stop won the final effect-admission transaction. */
 class EffectAdmissionFencedError extends Error {
   constructor(readonly effectId: string) {
@@ -173,6 +202,16 @@ class LoopAgent implements Agent {
   #cancelDetail: string | undefined;
   #disposeRequested = false;
   #resumeRequested = false;
+  /**
+   * The Turn's wall clock, rearmed for each Turn a wake runs.
+   *
+   * It aborts the same controller Stop uses, so nothing in the step loop has
+   * to learn about a second signal; the flag beside it is what tells the
+   * settlement that the abort was a deadline rather than a person.
+   */
+  #turnDeadlineTimer: ReturnType<typeof setTimeout> | undefined;
+  #turnDeadlineReached = false;
+  #turnDeadlineMs: number;
 
   constructor(
     ctx: Context,
@@ -180,6 +219,7 @@ class LoopAgent implements Agent {
     options: EffectAdmittingAgentOptions,
     maxSteps: number,
     composition: CompositionPinV1,
+    turnDeadlineMs: number,
   ) {
     this.#ctx = ctx;
     this.#composition = composition;
@@ -193,6 +233,7 @@ class LoopAgent implements Agent {
     this.#turnType = options.turnType ?? "chat";
     this.#subagentRole = options.subagentRole;
     this.#maxSteps = maxSteps;
+    this.#turnDeadlineMs = turnDeadlineMs;
   }
 
   get status(): AgentStatus {
@@ -255,6 +296,27 @@ class LoopAgent implements Agent {
     this.#controller?.abort(new Error(`agent cancelled by ${reason}`));
   }
 
+  /**
+   * Start this Turn's clock. Any previous Turn's is cleared first, so a wake
+   * that runs three queued Turns gives each of them the full allowance rather
+   * than sharing one.
+   */
+  #armTurnDeadline(): void {
+    this.#disarmTurnDeadline();
+    this.#turnDeadlineReached = false;
+    this.#turnDeadlineTimer = setTimeout(() => {
+      this.#turnDeadlineReached = true;
+      this.#controller?.abort(new Error(TURN_DEADLINE_REASON_V1));
+    }, this.#turnDeadlineMs);
+  }
+
+  #disarmTurnDeadline(): void {
+    if (this.#turnDeadlineTimer !== undefined) {
+      clearTimeout(this.#turnDeadlineTimer);
+      this.#turnDeadlineTimer = undefined;
+    }
+  }
+
   async whenIdle(): Promise<void> {
     let activity: Promise<void>;
     do {
@@ -384,6 +446,7 @@ class LoopAgent implements Agent {
     let turnOutcome: StepOutcome = "interrupted";
     let turnReason: string | undefined;
     let reconciliationRequired = false;
+    this.#armTurnDeadline();
     try {
       if (latestAssistant) {
         await this.#notifyModelOutcome(latestAssistant.requestId, "completed");
@@ -411,6 +474,19 @@ class LoopAgent implements Agent {
           latestStep,
           signal,
         );
+        if (reconciliation.status === "not-retrievable") {
+          // No later attempt can retrieve this effect, so the run settles now.
+          // The chunks already journaled stay in the session, so whatever the
+          // model produced before the interruption is still shown.
+          await this.#notifyModelOutcome(
+            unresolvedRequest.requestId,
+            "not-started",
+          );
+          turnOutcome = "model-error";
+          turnReason = turnEndReason(reconciliation.reason);
+          this.#ctx.emit("agent/error", this, new Error(reconciliation.reason));
+          return;
+        }
         if (reconciliation.status === "unavailable") {
           const existing = this.session.events.findLast(
             (event) =>
@@ -617,6 +693,13 @@ class LoopAgent implements Agent {
       ) {
         reconciliationRequired = true;
         this.#ctx.emit("agent/error", this, error);
+      } else if (this.#turnDeadlineReached) {
+        // Ahead of the cancellation branch on purpose: the deadline aborts the
+        // same controller Stop does, and a Turn the clock ended must not be
+        // reported to the person as one they stopped.
+        turnOutcome = "interrupted";
+        turnReason = turnEndReason(TURN_DEADLINE_REASON_V1);
+        this.#ctx.emit("agent/error", this, error);
       } else if (
         error instanceof EffectAdmissionFencedError ||
         signal.aborted
@@ -633,6 +716,13 @@ class LoopAgent implements Agent {
         this.#ctx.emit("agent/error", this, error);
       }
     } finally {
+      this.#disarmTurnDeadline();
+      // A Turn owed a reconciliation writes no `turn/end`: its model request
+      // has no durable outcome, and a `turn/end` would claim to know how it
+      // ended. That is right for as long as the run might still resume — and
+      // the moment it will not, the Turn is closed by whoever settles it, in
+      // `kernel-do`'s `settledEventsV1`. Closing it here instead would either
+      // lie about an outcome or make the run unresumable (ADR 0028).
       if (!reconciliationRequired) {
         if (openStep !== undefined && turnOutcome === "cancelled") {
           await this.#settleCancelledStep(openTurn, openStep);
@@ -685,6 +775,7 @@ class LoopAgent implements Agent {
     let turnOutcome: StepOutcome = "interrupted";
     let turnReason: string | undefined;
     let reconciliationRequired = false;
+    this.#armTurnDeadline();
     try {
       let inputs = [input];
       for (let step = 1; step <= this.#maxSteps; step += 1) {
@@ -784,6 +875,13 @@ class LoopAgent implements Agent {
       ) {
         reconciliationRequired = true;
         this.#ctx.emit("agent/error", this, error);
+      } else if (this.#turnDeadlineReached) {
+        // Ahead of the cancellation branch on purpose: the deadline aborts the
+        // same controller Stop does, and a Turn the clock ended must not be
+        // reported to the person as one they stopped.
+        turnOutcome = "interrupted";
+        turnReason = turnEndReason(TURN_DEADLINE_REASON_V1);
+        this.#ctx.emit("agent/error", this, error);
       } else if (
         error instanceof EffectAdmissionFencedError ||
         signal.aborted
@@ -800,6 +898,13 @@ class LoopAgent implements Agent {
         this.#ctx.emit("agent/error", this, error);
       }
     } finally {
+      this.#disarmTurnDeadline();
+      // A Turn owed a reconciliation writes no `turn/end`: its model request
+      // has no durable outcome, and a `turn/end` would claim to know how it
+      // ended. That is right for as long as the run might still resume — and
+      // the moment it will not, the Turn is closed by whoever settles it, in
+      // `kernel-do`'s `settledEventsV1`. Closing it here instead would either
+      // lie about an outcome or make the run unresumable (ADR 0028).
       if (!reconciliationRequired) {
         if (openStep !== undefined && turnOutcome === "cancelled") {
           await this.#settleCancelledStep(turn, openStep);
@@ -857,7 +962,16 @@ class LoopAgent implements Agent {
       turnType: this.#turnType,
     });
 
+    // One automatic retry, and only for a failure the provider itself
+    // classified as "the request never started" — a rejected key, an
+    // unresolvable binding, a connection refused before any byte was sent.
+    // Those are exactly the failures where retrying cannot duplicate anything,
+    // and the ones a person watching a blank screen would retry by hand. Every
+    // other failure is uncertain and is never retried, which is the whole of
+    // ADR 0024's durability contract.
+    let attempts = 0;
     while (true) {
+      attempts += 1;
       const proposedMessages = this.session.deriveMessages();
       const messages = await this.#ctx.waterfall(
         "agent/message-window",
@@ -962,14 +1076,24 @@ class LoopAgent implements Agent {
         });
         await this.session.flush();
         await this.#notifyModelOutcome(request.requestId, "not-started");
+        // The durable evidence of a retry is the log itself: a `model/request`,
+        // the `model/effect-not-started` just journaled against it, and then a
+        // second `model/request`. A Package listening on `agent/request-error`
+        // still has the final say in either direction.
         const action = await this.#ctx.waterfall(
           "agent/request-error",
           this,
           error,
           signal,
-          () => Promise.resolve({ kind: "fail" as const }),
+          () =>
+            Promise.resolve(
+              attempts < MODEL_REQUEST_ATTEMPTS_V1
+                ? ({ kind: "retry" } as const)
+                : ({ kind: "fail" } as const),
+            ),
         );
         if (action.kind !== "retry") throw error;
+        this.#ctx.emit("agent/error", this, error);
       }
     }
   }
@@ -1016,7 +1140,7 @@ class LoopAgent implements Agent {
     signal: AbortSignal,
   ): Promise<ModelReconciliation> {
     const reconciliation = await this.#ctx.llm.reconcile(request, signal);
-    if (reconciliation.status === "unavailable") return reconciliation;
+    if (reconciliation.status !== "recovered") return reconciliation;
     const durablePrefix = this.session.events.flatMap((event) =>
       event.type === "assistant/chunk" &&
       event.turn === turn &&
@@ -1313,6 +1437,7 @@ class LoopAgent implements Agent {
 export class AgentLoop extends Service implements AgentFactory {
   static inject = ["sessions", "systemPrompt", "llm", "tools", "agents"];
   private maxSteps: number;
+  private turnDeadlineMs: number;
   private composition: CompositionPinV1;
   private handles = new Set<AgentHandle>();
 
@@ -1320,6 +1445,7 @@ export class AgentLoop extends Service implements AgentFactory {
     super(ctx, "agentLoop");
     this.composition = config.composition;
     this.maxSteps = config.maxSteps ?? 20;
+    this.turnDeadlineMs = config.turnDeadlineMs ?? TURN_DEADLINE_MS_V1;
     if (!Number.isInteger(this.maxSteps) || this.maxSteps <= 0) {
       throw new Error("agent-loop maxSteps must be a positive integer");
     }
@@ -1336,6 +1462,7 @@ export class AgentLoop extends Service implements AgentFactory {
       options as EffectAdmittingAgentOptions,
       this.maxSteps,
       this.composition,
+      this.turnDeadlineMs,
     );
     const unregister = this.ctx.agents.register(agent);
     let disposed = false;