@frockbot/kernel-agent-loop 0.3.14 → 0.3.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/kernel-agent-loop",
-  "version": "0.3.14",
+  "version": "0.3.15",
   "private": false,
   "type": "module",
   "exports": {
@@ -12,13 +12,13 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-contracts": "0.3.14",
+    "@frockbot/kernel-contracts": "0.3.15",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {
-    "@frockbot/plugin-models": "0.3.14",
-    "@frockbot/plugin-prompt": "0.3.14",
-    "@frockbot/plugin-tools": "0.3.14",
+    "@frockbot/plugin-models": "0.3.15",
+    "@frockbot/plugin-prompt": "0.3.15",
+    "@frockbot/plugin-tools": "0.3.15",
     "@types/bun": "1.4.0",
     "@types/node": "26.2.0",
     "typescript": "^7.0.2"

package/src/agent.ts

@@ -61,7 +61,8 @@ export interface AgentSendV1 {
 export type PreStepDecision =
   { kind: "enter"; inputs: AgentInput[] } | { kind: "reject"; reason: string };
 
-export type RequestErrorAction = { kind: "retry" } | { kind: "fail" };
+export type RequestErrorAction =
+  { kind: "retry" } | { kind: "fallback" } | { kind: "fail" };
 
 export interface Agent extends LoopAgentRuntimeV1 {
   readonly id: string;

package/src/deadlines.test.ts

@@ -6,6 +6,7 @@ import { afterEach, describe, expect, test } from "bun:test";
 import {
   LlmEffectNotStartedError,
   type LlmProvider,
+  ModelProviderFailureError,
   SessionStore,
 } from "@frockbot/kernel-contracts";
 import { LlmRegistry } from "@frockbot/plugin-models";
@@ -24,7 +25,14 @@ afterEach(async () => {
 
 async function mount(
   provider: LlmProvider,
-  config: { turnDeadlineMs?: number } = {},
+  config: {
+    turnDeadlineMs?: number;
+    retry?: {
+      now?: () => number;
+      random?: () => number;
+      sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
+    };
+  } = {},
 ): Promise<Context> {
   const root = new Context();
   roots.push(root);
@@ -243,6 +251,11 @@ describe("a model request the provider says never started", () => {
         (event) => event.type === "model/effect-not-started",
       ),
     ).toHaveLength(1);
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/retry",
+      ),
+    ).toHaveLength(1);
   });
 
   test("is not retried a second time", async () => {
@@ -306,6 +319,121 @@ describe("a model request the provider says never started", () => {
   });
 });
 
+describe("classified model retry policy", () => {
+  test("backs off with injected time and random before a transient retry", async () => {
+    let attempts = 0;
+    let now = 10_000;
+    const sleeps: number[] = [];
+    const provider: LlmProvider = {
+      id: "transient-once",
+      async *stream() {
+        attempts += 1;
+        if (attempts === 1) {
+          throw new ModelProviderFailureError({
+            classification: "transient",
+            reason: "service unavailable",
+          });
+        }
+        yield { type: "text-delta", text: "Recovered." } as const;
+        yield { type: "finish", reason: "completed" } as const;
+      },
+    };
+    const root = await mount(provider, {
+      retry: {
+        now: () => now,
+        random: () => 1,
+        sleep: (milliseconds) => {
+          sleeps.push(milliseconds);
+          now += milliseconds;
+          return Promise.resolve();
+        },
+      },
+    });
+    const handle = await root.agents.create({
+      botId: "transient-bot",
+      sessionId: "transient",
+      provider: provider.id,
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Try the provider.");
+    await handle.agent.whenIdle();
+
+    expect(attempts).toBe(2);
+    expect(sleeps).toEqual([500]);
+    expect(
+      handle.agent.session.events.find((event) => event.type === "model/retry"),
+    ).toMatchObject({
+      attempt: 2,
+      classification: "transient",
+      delayMs: 500,
+    });
+  });
+
+  test("does not retry a permanent failure", async () => {
+    let attempts = 0;
+    const provider: LlmProvider = {
+      id: "permanent",
+      async *stream() {
+        attempts += 1;
+        throw new ModelProviderFailureError({
+          classification: "permanent",
+          reason: "credentials were refused",
+        });
+      },
+    };
+    const root = await mount(provider);
+    const handle = await root.agents.create({
+      botId: "permanent-bot",
+      sessionId: "permanent",
+      provider: provider.id,
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Try the provider.");
+    await handle.agent.whenIdle();
+
+    expect(attempts).toBe(1);
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/retry",
+      ),
+    ).toHaveLength(0);
+  });
+
+  test("does not schedule a retry beyond the remaining Turn deadline", async () => {
+    let attempts = 0;
+    const provider: LlmProvider = {
+      id: "deadline-bound",
+      async *stream() {
+        attempts += 1;
+        throw new ModelProviderFailureError({
+          classification: "transient",
+          reason: "service unavailable",
+        });
+      },
+    };
+    const root = await mount(provider, {
+      turnDeadlineMs: 400,
+      retry: { now: () => 0, random: () => 1 },
+    });
+    const handle = await root.agents.create({
+      botId: "bounded-bot",
+      sessionId: "bounded",
+      provider: provider.id,
+      model: "test-model",
+      admitEffect: allowEffect,
+    });
+
+    handle.agent.send("Try the provider.");
+    await handle.agent.whenIdle();
+
+    expect(attempts).toBe(1);
+  });
+});
+
 // Named so a reader who greps for the reason string finds where it is set.
 test("the deadline reason tells the person what to do about it", () => {
   expect(TURN_DEADLINE_REASON_V1).toContain("Try sending it again");

package/src/index.ts

@@ -15,6 +15,7 @@ import {
   type LlmStreamEvent,
   type LoopStepContinuationV1,
   type NormalizedModelRequest,
+  ModelProviderFailureError,
   type Session,
   type SessionEvent,
   type StepOutcome,
@@ -29,6 +30,13 @@ import {
   validateToolOccurrenceJournal,
 } from "@frockbot/kernel-contracts";
 import { type Context, Service } from "cordis";
+import {
+  defaultModelRetrySleepV1,
+  type ModelRetryPolicyRuntimeV1,
+  nextModelRetryV1,
+} from "./retry-policy.js";
+
+export * from "./retry-policy.js";
 
 declare module "cordis" {
   interface Events {
@@ -47,6 +55,8 @@ export interface AgentLoopConfig {
    * {@link TURN_DEADLINE_MS_V1}; named by a caller only to test it.
    */
   turnDeadlineMs?: number;
+  /** Deterministic retry seams; production uses wall time, Math.random and timers. */
+  retry?: Partial<ModelRetryPolicyRuntimeV1>;
   /** The Composition generation this mounted root was pinned to at admission. */
   composition: CompositionPinV1;
 }
@@ -120,13 +130,7 @@ class StepLimitReachedError extends Error {
  */
 export { TURN_DEADLINE_MS_V1 };
 
-/**
- * How many times one step will send its model request.
- *
- * Two: the first attempt and one retry. It applies only to a failure the
- * provider classified as never having started, so no retry can duplicate a
- * call that may already have run.
- */
+/** Legacy ceiling for unknown failures: the first attempt plus one retry. */
 export const MODEL_REQUEST_ATTEMPTS_V1 = 2;
 
 /** What a `turn/end` records when the Turn ran out of wall clock. */
@@ -213,6 +217,8 @@ class LoopAgent implements Agent {
   #turnDeadlineTimer: ReturnType<typeof setTimeout> | undefined;
   #turnDeadlineReached = false;
   #turnDeadlineMs: number;
+  #turnDeadlineAt = 0;
+  #retry: ModelRetryPolicyRuntimeV1;
 
   constructor(
     ctx: Context,
@@ -221,6 +227,7 @@ class LoopAgent implements Agent {
     maxSteps: number,
     composition: CompositionPinV1,
     turnDeadlineMs: number,
+    retry: ModelRetryPolicyRuntimeV1,
   ) {
     this.#ctx = ctx;
     this.#composition = composition;
@@ -235,6 +242,7 @@ class LoopAgent implements Agent {
     this.#subagentRole = options.subagentRole;
     this.#maxSteps = maxSteps;
     this.#turnDeadlineMs = turnDeadlineMs;
+    this.#retry = retry;
   }
 
   get status(): AgentStatus {
@@ -305,6 +313,7 @@ class LoopAgent implements Agent {
   #armTurnDeadline(): void {
     this.#disarmTurnDeadline();
     this.#turnDeadlineReached = false;
+    this.#turnDeadlineAt = this.#retry.now() + this.#turnDeadlineMs;
     this.#turnDeadlineTimer = setTimeout(() => {
       this.#turnDeadlineReached = true;
       this.#controller?.abort(new Error(TURN_DEADLINE_REASON_V1));
@@ -1060,7 +1069,7 @@ class LoopAgent implements Agent {
             reason,
           );
         }
-        if (!(error instanceof LlmEffectNotStartedError)) {
+        if (!(error instanceof ModelProviderFailureError)) {
           const reason = `Model response outcome is uncertain: ${modelFailureMessage(error)}`;
           this.session.append({
             type: "model/reconciliation-required",
@@ -1084,10 +1093,15 @@ class LoopAgent implements Agent {
         });
         await this.session.flush();
         await this.#notifyModelOutcome(request.requestId, "not-started");
-        // The durable evidence of a retry is the log itself: a `model/request`,
-        // the `model/effect-not-started` just journaled against it, and then a
-        // second `model/request`. A Package listening on `agent/request-error`
-        // still has the final say in either direction.
+        const retry = nextModelRetryV1({
+          failure: error,
+          attempt: attempts,
+          deadlineAt: this.#turnDeadlineAt,
+          runtime: this.#retry,
+        });
+        // A Package can refuse a planned retry, or replace a permanent failure
+        // with a provider-owned fallback. It cannot turn a permanent failure
+        // into another attempt against the same model.
         const action = await this.#ctx.waterfall(
           "agent/request-error",
           this,
@@ -1095,13 +1109,25 @@ class LoopAgent implements Agent {
           signal,
           () =>
             Promise.resolve(
-              attempts < MODEL_REQUEST_ATTEMPTS_V1
+              retry
                 ? ({ kind: "retry" } as const)
                 : ({ kind: "fail" } as const),
             ),
         );
-        if (action.kind !== "retry") throw error;
+        if (action.kind === "fail") throw error;
+        if (action.kind === "retry" && !retry) throw error;
+        const delayMs = action.kind === "fallback" ? 0 : retry!.delayMs;
+        this.session.append({
+          type: "model/retry",
+          turn,
+          step,
+          attempt: attempts + 1,
+          classification: error.classification,
+          delayMs,
+        });
+        await this.session.flush();
         this.#ctx.emit("agent/error", this, error);
+        await this.#retry.sleep(delayMs, signal);
       }
     }
   }
@@ -1131,9 +1157,10 @@ class LoopAgent implements Agent {
         );
       }
     } catch (error) {
-      if (receivedProviderEvent && error instanceof LlmEffectNotStartedError) {
+      if (receivedProviderEvent && error instanceof ModelProviderFailureError) {
         throw new Error(
-          "Model provider reported no effect after returning response data",
+          error.message ||
+            "Model provider reported a retryable failure after returning response data",
         );
       }
       throw error;
@@ -1495,6 +1522,7 @@ export class AgentLoop extends Service implements AgentFactory {
   private maxSteps: number;
   private turnDeadlineMs: number;
   private composition: CompositionPinV1;
+  private retry: ModelRetryPolicyRuntimeV1;
   private handles = new Set<AgentHandle>();
 
   constructor(ctx: Context, config: AgentLoopConfig) {
@@ -1502,6 +1530,11 @@ export class AgentLoop extends Service implements AgentFactory {
     this.composition = config.composition;
     this.maxSteps = config.maxSteps ?? 20;
     this.turnDeadlineMs = config.turnDeadlineMs ?? TURN_DEADLINE_MS_V1;
+    this.retry = {
+      now: config.retry?.now ?? Date.now,
+      random: config.retry?.random ?? Math.random,
+      sleep: config.retry?.sleep ?? defaultModelRetrySleepV1,
+    };
     if (!Number.isInteger(this.maxSteps) || this.maxSteps <= 0) {
       throw new Error("agent-loop maxSteps must be a positive integer");
     }
@@ -1519,6 +1552,7 @@ export class AgentLoop extends Service implements AgentFactory {
       this.maxSteps,
       this.composition,
       this.turnDeadlineMs,
+      this.retry,
     );
     const unregister = this.ctx.agents.register(agent);
     let disposed = false;

package/src/retry-policy.test.ts

@@ -0,0 +1,67 @@
+import { describe, expect, test } from "bun:test";
+import { ModelProviderFailureError } from "@frockbot/kernel-contracts";
+import {
+  MODEL_RETRY_BACKOFF_CAP_MS_V1,
+  modelRetryDelayV1,
+  nextModelRetryV1,
+} from "./retry-policy.js";
+
+describe("model retry backoff", () => {
+  test("uses deterministic equal-jitter exponential delays up to the cap", () => {
+    expect(
+      [1, 2, 3, 4, 5, 6].map((retry) =>
+        modelRetryDelayV1({ retry, random: 1 }),
+      ),
+    ).toEqual([500, 1_000, 2_000, 4_000, 8_000, 8_000]);
+    expect(modelRetryDelayV1({ retry: 1, random: 0 })).toBe(250);
+    expect(MODEL_RETRY_BACKOFF_CAP_MS_V1).toBe(8_000);
+  });
+
+  test("honours Retry-After even when it is longer than the backoff cap", () => {
+    expect(
+      modelRetryDelayV1({ retry: 6, random: 0, retryAfterMs: 12_000 }),
+    ).toBe(12_000);
+  });
+
+  test("allows unknown once, permanent never, and transient while time remains", () => {
+    let now = 1_000;
+    const runtime = { now: () => now, random: () => 1 };
+    const failure = (classification: "transient" | "permanent" | "unknown") =>
+      new ModelProviderFailureError({ classification, reason: "test" });
+
+    expect(
+      nextModelRetryV1({
+        failure: failure("unknown"),
+        attempt: 1,
+        deadlineAt: 2_000,
+        runtime,
+      }),
+    ).toEqual({ attempt: 2, delayMs: 500 });
+    expect(
+      nextModelRetryV1({
+        failure: failure("unknown"),
+        attempt: 2,
+        deadlineAt: 10_000,
+        runtime,
+      }),
+    ).toBeUndefined();
+    expect(
+      nextModelRetryV1({
+        failure: failure("permanent"),
+        attempt: 1,
+        deadlineAt: 10_000,
+        runtime,
+      }),
+    ).toBeUndefined();
+
+    now = 1_600;
+    expect(
+      nextModelRetryV1({
+        failure: failure("transient"),
+        attempt: 1,
+        deadlineAt: 2_000,
+        runtime,
+      }),
+    ).toBeUndefined();
+  });
+});

package/src/retry-policy.ts

@@ -0,0 +1,83 @@
+import type {
+  ModelProviderFailureClassV1,
+  ModelProviderFailureError,
+} from "@frockbot/kernel-contracts";
+
+export const MODEL_RETRY_BACKOFF_BASE_MS_V1 = 500;
+export const MODEL_RETRY_BACKOFF_CAP_MS_V1 = 8_000;
+
+export interface ModelRetryPolicyRuntimeV1 {
+  now(): number;
+  random(): number;
+  sleep(milliseconds: number, signal: AbortSignal): Promise<void>;
+}
+
+export function defaultModelRetrySleepV1(
+  milliseconds: number,
+  signal: AbortSignal,
+): Promise<void> {
+  if (milliseconds === 0) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(done, milliseconds);
+    function done(): void {
+      signal.removeEventListener("abort", aborted);
+      resolve();
+    }
+    function aborted(): void {
+      clearTimeout(timer);
+      reject(signal.reason);
+    }
+    if (signal.aborted) aborted();
+    else signal.addEventListener("abort", aborted, { once: true });
+  });
+}
+
+/** Equal-jitter exponential delay; Retry-After is a floor, not silently capped. */
+export function modelRetryDelayV1(input: {
+  retry: number;
+  random: number;
+  retryAfterMs?: number;
+}): number {
+  const ceiling = Math.min(
+    MODEL_RETRY_BACKOFF_CAP_MS_V1,
+    MODEL_RETRY_BACKOFF_BASE_MS_V1 * 2 ** Math.max(0, input.retry - 1),
+  );
+  const boundedRandom = Math.min(1, Math.max(0, input.random));
+  const jittered = Math.round(ceiling / 2 + (ceiling / 2) * boundedRandom);
+  return Math.max(jittered, input.retryAfterMs ?? 0);
+}
+
+export function modelFailureMayRetryV1(input: {
+  classification: ModelProviderFailureClassV1;
+  attempt: number;
+}): boolean {
+  if (input.classification === "transient") return true;
+  return input.classification === "unknown" && input.attempt === 1;
+}
+
+/** Undefined means the Turn has too little wall clock left for another try. */
+export function nextModelRetryV1(input: {
+  failure: ModelProviderFailureError;
+  attempt: number;
+  deadlineAt: number;
+  runtime: Pick<ModelRetryPolicyRuntimeV1, "now" | "random">;
+}): { attempt: number; delayMs: number } | undefined {
+  if (
+    !modelFailureMayRetryV1({
+      classification: input.failure.classification,
+      attempt: input.attempt,
+    })
+  ) {
+    return undefined;
+  }
+  const delayMs = modelRetryDelayV1({
+    retry: input.attempt,
+    random: input.runtime.random(),
+    ...(input.failure.retryAfterMs === undefined
+      ? {}
+      : { retryAfterMs: input.failure.retryAfterMs }),
+  });
+  return delayMs < input.deadlineAt - input.runtime.now()
+    ? { attempt: input.attempt + 1, delayMs }
+    : undefined;
+}