@frockbot/kernel-agent-loop 0.3.16 → 0.3.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/kernel-agent-loop",
-  "version": "0.3.16",
+  "version": "0.3.18",
   "private": false,
   "type": "module",
   "exports": {
@@ -12,13 +12,13 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-contracts": "0.3.16",
+    "@frockbot/kernel-contracts": "0.3.18",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {
-    "@frockbot/plugin-models": "0.3.16",
-    "@frockbot/plugin-prompt": "0.3.16",
-    "@frockbot/plugin-tools": "0.3.16",
+    "@frockbot/plugin-models": "0.3.18",
+    "@frockbot/plugin-prompt": "0.3.18",
+    "@frockbot/plugin-tools": "0.3.18",
     "@types/bun": "1.4.0",
     "@types/node": "26.2.0",
     "typescript": "^7.0.2"

package/src/agent.ts

@@ -126,7 +126,17 @@ declare module "cordis" {
     "agent/assistant-text": (
       agent: Agent,
       text: string,
-      position: { turn: number; step: number; requestId: string },
+      position: {
+        turn: number;
+        step: number;
+        requestId: string;
+        /**
+         * The tools the same step is about to call, by name, so a listener
+         * can tell an acknowledgement the model *only* narrated from one it
+         * is also delivering through its own send tool.
+         */
+        toolNames?: readonly string[];
+      },
     ) => Promise<void>;
   }
 }

package/src/index.test.ts

@@ -17,7 +17,7 @@ import { SystemPromptRegistry } from "@frockbot/plugin-prompt";
 import { ToolRegistry } from "@frockbot/plugin-tools";
 import { AgentRegistry, type AgentOptions } from "./agent.js";
 import { Context, type Plugin } from "cordis";
-import { AgentLoop } from "./index.js";
+import { AgentLoop, STEP_LIMIT_REASON_V1 } from "./index.js";
 
 const roots: Context[] = [];
 const allowEffect = () => Promise.resolve(true);
@@ -151,6 +151,170 @@ afterEach(async () => {
 });
 
 describe("AgentLoop", () => {
+  test("journals one reported usage event for a model request", async () => {
+    const provider: LlmProvider = {
+      id: "reported-usage",
+      async *stream() {
+        yield { type: "text-delta", text: "done" };
+        yield {
+          type: "usage",
+          usage: {
+            inputTokens: 41,
+            outputTokens: 9,
+            cachedInputTokens: 7,
+            reasoningTokens: 3,
+          },
+        };
+        yield { type: "finish", reason: "completed" };
+      },
+    };
+    const root = await mountRuntime(provider);
+    const handle = await root.agents.create({
+      ...allowEffectOptions,
+      botId: "bot-usage",
+      sessionId: "reported-usage",
+      provider: provider.id,
+      model: "priced-model",
+    });
+
+    handle.agent.send("count this");
+    await handle.agent.whenIdle();
+
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/usage",
+      ),
+    ).toEqual([
+      expect.objectContaining({
+        type: "model/usage",
+        provider: "reported-usage",
+        model: "priced-model",
+        inputTokens: 41,
+        outputTokens: 9,
+        cachedInputTokens: 7,
+        reasoningTokens: 3,
+        estimated: false,
+      }),
+    ]);
+  });
+
+  test("estimates and marks usage when a provider reports no counts", async () => {
+    const provider: LlmProvider = {
+      id: "estimated-usage",
+      async *stream() {
+        yield { type: "text-delta", text: "an answer" };
+        yield { type: "finish", reason: "completed" };
+      },
+    };
+    const root = await mountRuntime(provider);
+    const handle = await root.agents.create({
+      ...allowEffectOptions,
+      botId: "bot-usage",
+      sessionId: "estimated-usage",
+      provider: provider.id,
+      model: "unmetered-model",
+    });
+
+    handle.agent.send("count approximately");
+    await handle.agent.whenIdle();
+
+    const usage = handle.agent.session.events.filter(
+      (event) => event.type === "model/usage",
+    );
+    expect(usage).toHaveLength(1);
+    expect(usage[0]).toMatchObject({
+      type: "model/usage",
+      provider: "estimated-usage",
+      model: "unmetered-model",
+      estimated: true,
+    });
+    if (usage[0]?.type !== "model/usage") throw new Error("usage missing");
+    expect(usage[0].inputTokens).toBeGreaterThan(0);
+    expect(usage[0].outputTokens).toBeGreaterThan(0);
+  });
+
+  test("journals a structured-output downgrade and typed validation failure", async () => {
+    const provider: LlmProvider = {
+      id: "structured-failure",
+      supports: { structuredOutput: "none" },
+      async *stream() {
+        yield {
+          type: "response-format-note",
+          note: {
+            code: "structured-output-downgraded",
+            requested: "json_schema",
+            effective: "prompt",
+            message: "Fake provider used prompt guidance",
+          },
+        };
+        yield { type: "text-delta", text: '{"answer":4}' };
+        yield { type: "finish", reason: "completed" };
+      },
+    };
+    const root = await mountRuntime(provider);
+    root.on("agent/request", async (_agent, _request, _signal, next) => ({
+      ...(await next()),
+      responseFormat: {
+        type: "json_schema",
+        name: "answer",
+        schema: {
+          type: "object",
+          properties: { answer: { type: "string" } },
+          required: ["answer"],
+          additionalProperties: false,
+        },
+      },
+    }));
+    const handle = await root.agents.create({
+      ...allowEffectOptions,
+      botId: "bot-structured-failure",
+      sessionId: "structured-failure",
+      provider: provider.id,
+      model: "fake",
+    });
+
+    handle.agent.send("answer as an object");
+    await handle.agent.whenIdle();
+
+    expect(
+      handle.agent.session.events.find(
+        (event) => event.type === "model/response-format-note",
+      ),
+    ).toMatchObject({
+      type: "model/response-format-note",
+      note: { effective: "prompt" },
+    });
+    expect(
+      handle.agent.session.events.find(
+        (event) => event.type === "model/response-failed",
+      ),
+    ).toMatchObject({
+      type: "model/response-failed",
+      failure: { code: "schema-mismatch" },
+    });
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/usage",
+      ),
+    ).toEqual([
+      expect.objectContaining({
+        type: "model/usage",
+        provider: "structured-failure",
+        estimated: true,
+      }),
+    ]);
+    expect(
+      handle.agent.session.events.some(
+        (event) => event.type === "model/reconciliation-required",
+      ),
+    ).toBe(false);
+    expect(
+      handle.agent.session.events.findLast(
+        (event) => event.type === "turn/end",
+      ),
+    ).toMatchObject({ type: "turn/end", outcome: "model-error" });
+  });
+
   test("records exactly the hook-shaped request received by the provider", async () => {
     let received: NormalizedModelRequest | undefined;
     const provider: LlmProvider = {
@@ -1355,6 +1519,16 @@ describe("AgentLoop", () => {
           "Model response outcome is uncertain: response lost after dispatch",
       }),
     );
+    expect(
+      handle.agent.session.events.filter(
+        (event) => event.type === "model/usage",
+      ),
+    ).toEqual([
+      expect.objectContaining({
+        requestId: request.request.requestId,
+        estimated: true,
+      }),
+    ]);
     expect(
       handle.agent.session.events.some(
         (event) => event.type === "step/end" || event.type === "turn/end",
@@ -3008,7 +3182,7 @@ describe("AgentLoop", () => {
     expect(handle.agent.session.events.at(-1)).toMatchObject({
       type: "turn/end",
       outcome: "interrupted",
-      reason: "stopped after 4 steps",
+      reason: STEP_LIMIT_REASON_V1,
     });
     // Nothing about the model failed, so nothing is reported as if it had.
     expect(errors).toEqual([]);
@@ -3087,6 +3261,7 @@ describe("AgentLoop", () => {
     root.on("agent/assistant-text", async (_agent, text, position) => {
       order.push("assistant-text");
       seen.push(`${position.turn}:${position.step}:${text}`);
+      seen.push(`tools:${(position.toolNames ?? []).join(",")}`);
     });
     const handle = await root.agents.create({
       ...allowEffectOptions,
@@ -3100,6 +3275,7 @@ describe("AgentLoop", () => {
     await handle.agent.whenIdle();
 
     expect(seen[0]).toBe("1:1:On it — building it now.");
+    expect(seen[1]).toBe("tools:build");
     expect(order[0]).toBe("assistant-text");
     expect(order).toContain("tool");
   });

package/src/index.ts

@@ -13,12 +13,14 @@ import {
   decodeSkillRefsV1,
   LlmEffectNotStartedError,
   type LlmStreamEvent,
+  type LlmUsageV1,
   type LoopStepContinuationV1,
   type NormalizedModelRequest,
   ModelProviderFailureError,
   type Session,
   type SessionEvent,
   type StepOutcome,
+  StructuredOutputValidationError,
   type ToolCall,
   type ToolCallOccurrence,
   type ToolExecutionResult,
@@ -80,6 +82,31 @@ interface ModelResponse {
   toolCalls: ToolCall[];
 }
 
+const TOKEN_ESTIMATE_BYTES_PER_TOKEN_V1 = 4;
+
+function estimatedTokensV1(value: unknown): number {
+  const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
+  return Math.ceil(bytes / TOKEN_ESTIMATE_BYTES_PER_TOKEN_V1);
+}
+
+/**
+ * The provider-neutral fallback for transports that return no token counts.
+ * It is intentionally based on the exact normalized request and assembled
+ * response that are journaled, and is always marked estimated at the event.
+ */
+export function estimateModelUsageV1(
+  request: NormalizedModelRequest,
+  response: Pick<ModelResponse, "text" | "toolCalls">,
+): LlmUsageV1 {
+  return {
+    inputTokens: estimatedTokensV1(request),
+    outputTokens: estimatedTokensV1({
+      text: response.text,
+      toolCalls: response.toolCalls,
+    }),
+  };
+}
+
 type ModelReconciliation =
   | { status: "recovered"; response: ModelResponse }
   | { status: "unavailable"; reason: string }
@@ -115,11 +142,24 @@ class ToolEffectReconciliationRequiredError extends Error {
  */
 class StepLimitReachedError extends Error {
   constructor(readonly steps: number) {
-    super(`stopped after ${steps} steps`);
+    // The sentence is written for the person, the way the Turn deadline's is:
+    // it is what reaches the chat bubble once `kernel-do` wraps it into the
+    // run's failure. The step count stays on the error for the log.
+    super(STEP_LIMIT_REASON_V1);
     this.name = "StepLimitReachedError";
   }
 }
 
+/**
+ * What a person is told when a reply used every step it was allowed. Bob
+ * (2026-09-04) ran a to-do applet build to the 64-step ceiling and the thread
+ * showed the generic "stopped before it finished" under a spinner that never
+ * ended; this names what happened and what to do, and stays true whatever
+ * the ceiling is.
+ */
+export const STEP_LIMIT_REASON_V1 =
+  "This Bot used all the steps it had for one reply and stopped. What it finished is saved. Send another message to carry on.";
+
 /**
  * The longest a single Turn may run before the loop stops waiting for it.
  *
@@ -397,6 +437,8 @@ class LoopAgent implements Agent {
     let unresolvedRequest: NormalizedModelRequest | undefined;
     let definitiveNoEffect:
       Extract<SessionEvent, { type: "model/effect-not-started" }> | undefined;
+    let definitiveResponseFailure:
+      Extract<SessionEvent, { type: "model/response-failed" }> | undefined;
     for (const event of this.session.events) {
       if (event.type === "turn/start") {
         openTurn = event.turn;
@@ -405,6 +447,7 @@ class LoopAgent implements Agent {
         latestStepOutcome = undefined;
         unresolvedRequest = undefined;
         definitiveNoEffect = undefined;
+        definitiveResponseFailure = undefined;
       }
       if (event.type === "turn/end" && event.turn === openTurn)
         openTurn = undefined;
@@ -424,6 +467,7 @@ class LoopAgent implements Agent {
       if (event.type === "model/request" && event.turn === openTurn) {
         unresolvedRequest = event.request;
         definitiveNoEffect = undefined;
+        definitiveResponseFailure = undefined;
       }
       if (
         event.type === "model/effect-not-started" &&
@@ -431,6 +475,12 @@ class LoopAgent implements Agent {
       ) {
         definitiveNoEffect = event;
       }
+      if (
+        event.type === "model/response-failed" &&
+        event.requestId === unresolvedRequest?.requestId
+      ) {
+        definitiveResponseFailure = event;
+      }
       if (
         event.type === "assistant/message" &&
         event.requestId === unresolvedRequest?.requestId
@@ -464,6 +514,22 @@ class LoopAgent implements Agent {
       let nextStep = latestStep === 0 ? 1 : latestStep + 1;
       if (unresolvedRequest) {
         openStep = latestStep;
+        if (definitiveResponseFailure) {
+          await this.#notifyModelOutcome(
+            definitiveResponseFailure.requestId,
+            "completed",
+          );
+          turnOutcome = "model-error";
+          turnReason = turnEndReason(definitiveResponseFailure.failure.message);
+          this.#ctx.emit(
+            "agent/error",
+            this,
+            new StructuredOutputValidationError(
+              definitiveResponseFailure.failure,
+            ),
+          );
+          return;
+        }
         if (definitiveNoEffect) {
           await this.#notifyModelOutcome(
             definitiveNoEffect.requestId,
@@ -977,6 +1043,9 @@ class LoopAgent implements Agent {
       // The same turn type the tool catalog is trimmed to. A section that
       // renders what a Turn may do would otherwise have to guess it.
       turnType: this.#turnType,
+      // Where the Turn is in its budget, so a section can warn the model
+      // before the loop stops it.
+      step: { current: step, max: this.#maxSteps },
     });
 
     // One automatic retry, and only for a failure the provider itself
@@ -1054,6 +1123,11 @@ class LoopAgent implements Agent {
       try {
         return await this.#consumeStream(request, turn, step, signal);
       } catch (error) {
+        if (error instanceof StructuredOutputValidationError) {
+          await this.session.flush();
+          await this.#notifyModelOutcome(request.requestId, "completed");
+          throw error;
+        }
         if (signal.aborted) {
           const reason = `Model response outcome is uncertain after cancellation: ${modelFailureMessage(error)}`;
           this.session.append({
@@ -1140,11 +1214,20 @@ class LoopAgent implements Agent {
   ): Promise<ModelResponse> {
     let text = "";
     const toolCalls: ToolCall[] = [];
+    let usage: LlmUsageV1 | undefined;
+    let structuredFailure:
+      | Extract<
+          LlmStreamEvent,
+          { type: "structured-output-failure" }
+        >["failure"]
+      | undefined;
     let receivedProviderEvent = false;
+    const startedAt = Date.now();
     try {
       for await (const event of this.#ctx.llm.stream(request, signal)) {
-        receivedProviderEvent = true;
+        if (event.type !== "response-format-note") receivedProviderEvent = true;
         signal.throwIfAborted();
+        if (event.type === "usage") usage = structuredClone(event.usage);
         this.#applyStreamEvent(
           event,
           request.requestId,
@@ -1155,16 +1238,57 @@ class LoopAgent implements Agent {
             text += delta;
           },
         );
+        if (event.type === "structured-output-failure") {
+          structuredFailure = event.failure;
+        }
       }
     } catch (error) {
       if (receivedProviderEvent && error instanceof ModelProviderFailureError) {
-        throw new Error(
+        const invalidNoEffectClaim = new Error(
           error.message ||
             "Model provider reported a retryable failure after returning response data",
         );
+        this.#recordModelUsage(
+          request,
+          turn,
+          step,
+          usage,
+          text,
+          toolCalls,
+          Math.max(0, Date.now() - startedAt),
+        );
+        throw invalidNoEffectClaim;
+      }
+      // Once dispatch may have begun, the call can have incurred spend even
+      // when its terminal response is lost. Preserve the provider's partial
+      // counts when present and otherwise write the same explicit estimate as
+      // a successful unmetered stream. A definitive no-effect result is the
+      // sole exception because the provider says no billable call occurred.
+      if (!(error instanceof ModelProviderFailureError)) {
+        this.#recordModelUsage(
+          request,
+          turn,
+          step,
+          usage,
+          text,
+          toolCalls,
+          Math.max(0, Date.now() - startedAt),
+        );
       }
       throw error;
     }
+    this.#recordModelUsage(
+      request,
+      turn,
+      step,
+      usage,
+      text,
+      toolCalls,
+      Math.max(0, Date.now() - startedAt),
+    );
+    if (structuredFailure) {
+      throw new StructuredOutputValidationError(structuredFailure);
+    }
     return { request, text, toolCalls };
   }
 
@@ -1211,9 +1335,18 @@ class LoopAgent implements Agent {
     }
     let text = "";
     const toolCalls: ToolCall[] = [];
+    let usage: LlmUsageV1 | undefined;
+    let structuredFailure:
+      | Extract<
+          LlmStreamEvent,
+          { type: "structured-output-failure" }
+        >["failure"]
+      | undefined;
     let textDeltaIndex = 0;
+    const startedAt = Date.now();
     for (const event of reconciliation.events) {
       signal.throwIfAborted();
+      if (event.type === "usage") usage = structuredClone(event.usage);
       const journalTextDelta =
         event.type !== "text-delta" || textDeltaIndex >= durablePrefix.length;
       this.#applyStreamEvent(
@@ -1228,6 +1361,23 @@ class LoopAgent implements Agent {
         journalTextDelta,
       );
       if (event.type === "text-delta") textDeltaIndex += 1;
+      if (event.type === "structured-output-failure") {
+        structuredFailure = event.failure;
+      }
+    }
+    this.#recordModelUsage(
+      request,
+      turn,
+      step,
+      usage,
+      text,
+      toolCalls,
+      Math.max(0, Date.now() - startedAt),
+    );
+    if (structuredFailure) {
+      await this.session.flush();
+      await this.#notifyModelOutcome(request.requestId, "completed");
+      throw new StructuredOutputValidationError(structuredFailure);
     }
     return {
       status: "recovered",
@@ -1235,6 +1385,39 @@ class LoopAgent implements Agent {
     };
   }
 
+  #recordModelUsage(
+    request: NormalizedModelRequest,
+    turn: number,
+    step: number,
+    reported: LlmUsageV1 | undefined,
+    text: string,
+    toolCalls: readonly ToolCall[],
+    latencyMs: number,
+  ): void {
+    const existing = this.session.events.some(
+      (event) =>
+        event.type === "model/usage" && event.requestId === request.requestId,
+    );
+    if (existing) return;
+    const usage =
+      reported ??
+      estimateModelUsageV1(request, { text, toolCalls: [...toolCalls] });
+    this.session.append({
+      type: "model/usage",
+      turn,
+      step,
+      requestId: request.requestId,
+      provider: request.provider,
+      model: request.model,
+      ...(request.modelBinding
+        ? { modelBinding: structuredClone(request.modelBinding) }
+        : {}),
+      ...usage,
+      latencyMs,
+      estimated: reported === undefined,
+    });
+  }
+
   #applyStreamEvent(
     event: LlmStreamEvent,
     requestId: string,
@@ -1257,6 +1440,22 @@ class LoopAgent implements Agent {
       }
     } else if (event.type === "tool-call") {
       toolCalls.push(event.call);
+    } else if (event.type === "response-format-note") {
+      this.session.append({
+        type: "model/response-format-note",
+        turn,
+        step,
+        requestId,
+        note: event.note,
+      });
+    } else if (event.type === "structured-output-failure") {
+      this.session.append({
+        type: "model/response-failed",
+        turn,
+        step,
+        requestId,
+        failure: event.failure,
+      });
     }
   }
 
@@ -1424,6 +1623,7 @@ class LoopAgent implements Agent {
       turn,
       step,
       requestId: response.request.requestId,
+      toolNames: response.toolCalls.map((call) => call.name),
     });
   }