@vellumai/assistant 0.8.5 → 0.8.7

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -14,14 +14,17 @@
  * require deeper provider fakery and are best covered by integration tests
  * once we wire up the empty-response pipeline mock.
  */
-import { describe, expect, test } from "bun:test";
+import { describe, expect, spyOn, test } from "bun:test";
 
 import type {
   AgentEvent,
   CheckpointDecision,
   CheckpointInfo,
+  MidLoopCompaction,
 } from "../agent/loop.js";
-import { AgentLoop } from "../agent/loop.js";
+import { AgentLoop, isMaxTokensStopReason } from "../agent/loop.js";
+import type { TurnContext } from "../plugins/types.js";
+import { PluginTimeoutError } from "../plugins/types.js";
 import type {
   Message,
   Provider,
@@ -42,8 +45,6 @@ function createMockProvider(responses: ProviderResponse[]): {
     name: "mock",
     async sendMessage(
       _messages: Message[],
-      _tools?: ToolDefinition[],
-      _systemPrompt?: string,
       _options?: SendMessageOptions,
     ): Promise<ProviderResponse> {
       const response = responses[callIndex] ?? responses[responses.length - 1];
@@ -63,6 +64,18 @@ function textResponse(text: string): ProviderResponse {
   };
 }
 
+function maxTokensResponse(
+  text: string,
+  stopReason: string = "max_tokens",
+): ProviderResponse {
+  return {
+    content: [{ type: "text", text }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason,
+  };
+}
+
 function toolUseResponse(
   id: string,
   name: string,
@@ -89,6 +102,40 @@ const userMessage: Message = {
   content: [{ type: "text", text: "Hello" }],
 };
 
+// A turn context whose `contextWindowManager.maybeCompact` returns a canned
+// result, so the loop's native compaction pipeline runs without the real
+// orchestrator machinery.
+function fakeCompactionTurnContext(result: {
+  compacted: boolean;
+  exhausted: boolean;
+}): TurnContext {
+  return {
+    requestId: "req-compact",
+    conversationId: "conv-compact",
+    turnIndex: 0,
+    trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    contextWindowManager: {
+      maybeCompact: async () => result,
+    },
+  } as unknown as TurnContext;
+}
+
+// A turn context whose compaction call times out, exercising the loop's
+// PluginTimeoutError handling.
+function timeoutCompactionTurnContext(): TurnContext {
+  return {
+    requestId: "req-compact",
+    conversationId: "conv-compact",
+    turnIndex: 0,
+    trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    contextWindowManager: {
+      maybeCompact: async () => {
+        throw new PluginTimeoutError("compaction", "default-compaction", 1);
+      },
+    },
+  } as unknown as TurnContext;
+}
+
 function lastExitEvent(
   events: AgentEvent[],
 ): Extract<AgentEvent, { type: "agent_loop_exit" }> | undefined {
@@ -107,6 +154,15 @@ function countExitEvents(events: AgentEvent[]): number {
 // ---------------------------------------------------------------------------
 
 describe("AgentLoop exit-reason instrumentation", () => {
+  test("recognizes provider output-token stop reasons", () => {
+    expect(isMaxTokensStopReason("max_tokens")).toBe(true);
+    expect(isMaxTokensStopReason("MAX_TOKENS")).toBe(true);
+    expect(isMaxTokensStopReason("length")).toBe(true);
+    expect(isMaxTokensStopReason("max_output_tokens")).toBe(true);
+    expect(isMaxTokensStopReason("end_turn")).toBe(false);
+    expect(isMaxTokensStopReason(undefined)).toBe(false);
+  });
+
   test("emits exit event exactly once with 'no_tool_calls' on plain text response", async () => {
     const { provider } = createMockProvider([textResponse("Hi there!")]);
     const loop = new AgentLoop(provider, "system prompt");
@@ -134,6 +190,61 @@ describe("AgentLoop exit-reason instrumentation", () => {
     expect(events[events.length - 1].type).toBe("agent_loop_exit");
   });
 
+  test("emits continuation surface event and exits on max_tokens", async () => {
+    const { provider } = createMockProvider([
+      maxTokensResponse("Partial answer"),
+    ]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(events.map((e) => e.type)).toEqual([
+      "llm_call_started",
+      "usage",
+      "max_tokens_reached",
+      "message_complete",
+      "agent_loop_exit",
+    ]);
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("max_tokens_reached");
+  });
+
+  test("does not persist unexecuted tool_use blocks when max_tokens stops output", async () => {
+    const { provider } = createMockProvider([
+      {
+        content: [
+          { type: "text", text: "I need to check that." },
+          {
+            type: "tool_use",
+            id: "tool-1",
+            name: "read_file",
+            input: { path: "/tmp/example.txt" },
+          },
+        ],
+        model: "mock-model",
+        usage: { inputTokens: 10, outputTokens: 5 },
+        stopReason: "max_tokens",
+      },
+    ]);
+    const loop = new AgentLoop(provider, "system prompt", {
+      tools: dummyTools,
+    });
+
+    const events: AgentEvent[] = [];
+    const { history: result } = await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(events.some((e) => e.type === "tool_use")).toBe(false);
+    expect(lastExitEvent(events)?.reason).toBe("max_tokens_reached");
+    expect(result[result.length - 1]!.content).toEqual([
+      { type: "text", text: "I need to check that." },
+    ]);
+  });
+
   test("emits 'aborted_pre_call' when signal is already aborted at run start", async () => {
     const { provider } = createMockProvider([textResponse("never sent")]);
     const loop = new AgentLoop(provider, "system prompt");
@@ -142,7 +253,13 @@ describe("AgentLoop exit-reason instrumentation", () => {
     controller.abort();
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+    await loop.run(
+      [userMessage],
+      (e) => {
+        events.push(e);
+      },
+      { signal: controller.signal },
+    );
 
     expect(countExitEvents(events)).toBe(1);
     expect(lastExitEvent(events)?.reason).toBe("aborted_pre_call");
@@ -157,16 +274,15 @@ describe("AgentLoop exit-reason instrumentation", () => {
       isError: false,
       yieldToUser: true,
     });
-    const loop = new AgentLoop(
-      provider,
-      "system",
-      {},
-      dummyTools,
-      toolExecutor,
-    );
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => { events.push(e); });
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
 
     expect(countExitEvents(events)).toBe(1);
     expect(lastExitEvent(events)?.reason).toBe("yield_to_user");
@@ -178,29 +294,203 @@ describe("AgentLoop exit-reason instrumentation", () => {
       textResponse("never reached"),
     ]);
     const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(
-      provider,
-      "system",
-      {},
-      dummyTools,
-      toolExecutor,
-    );
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
 
     const onCheckpoint = (_info: CheckpointInfo): CheckpointDecision =>
-      "yield";
+      "budget";
 
     const events: AgentEvent[] = [];
     await loop.run(
       [userMessage],
-      (e) => { events.push(e); },
-      undefined,
-      undefined,
-      onCheckpoint,
+      (e) => {
+        events.push(e);
+      },
+      { onCheckpoint },
     );
 
     expect(countExitEvents(events)).toBe(0);
   });
 
+  test("yields with 'budget' when the in-loop budget gate trips", async () => {
+    // GIVEN a provider that calls a tool (reaching a checkpoint) before it
+    // would continue with a text response.
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
+
+    // AND an effective context window so small that any real token estimate
+    // of the running history exceeds the mid-loop threshold.
+    // WHEN the loop checkpoints after the tool results land
+    const result = await loop.run([userMessage], () => {}, {
+      resolveContextWindow: () => ({
+        maxInputTokens: 10,
+        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
+      }),
+    });
+
+    // THEN it yields for budget before issuing the next provider call.
+    expect(result.exitReason).toBe("budget");
+  });
+
+  test("does not yield for budget when overflow recovery is disabled", async () => {
+    // GIVEN the same tiny context window but overflow recovery disabled — the
+    // agent-wake configuration, which must never yield for budget.
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("done"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
+
+    // WHEN the loop runs to completion
+    const result = await loop.run([userMessage], () => {}, {
+      resolveContextWindow: () => ({
+        maxInputTokens: 10,
+        overflowRecovery: { enabled: false, safetyMarginRatio: 0 },
+      }),
+    });
+
+    // THEN it never yields for budget.
+    expect(result.exitReason).not.toBe("budget");
+  });
+
+  test("compacts in place and continues when the budget gate trips with a compaction hook", async () => {
+    // GIVEN a tool round that reaches a checkpoint, followed by a plain text
+    // response that ends the run after compaction continues the loop.
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("done after compaction"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
+
+    let prepared = false;
+    let appliedResult = false;
+    let reinjected = false;
+    const compaction: MidLoopCompaction = {
+      prepare: (history) => {
+        prepared = true;
+        return { rawHistory: history, options: undefined };
+      },
+      applyResult: async () => {
+        appliedResult = true;
+      },
+      reinject: async () => {
+        reinjected = true;
+        return [userMessage];
+      },
+    };
+
+    // WHEN the in-loop budget gate trips at the checkpoint
+    const result = await loop.run([userMessage], () => {}, {
+      resolveContextWindow: () => ({
+        maxInputTokens: 10,
+        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
+      }),
+      compaction,
+      turnContext: fakeCompactionTurnContext({
+        compacted: true,
+        exhausted: false,
+      }),
+    });
+
+    // THEN the loop runs the compaction ceremony in place and continues to a
+    // clean exit instead of yielding for budget.
+    expect(prepared).toBe(true);
+    expect(appliedResult).toBe(true);
+    expect(reinjected).toBe(true);
+    expect(result.exitReason).not.toBe("budget");
+  });
+
+  test("yields 'budget' when inline compaction times out", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
+
+    const compaction: MidLoopCompaction = {
+      prepare: (history) => ({ rawHistory: history, options: undefined }),
+      applyResult: async () => {},
+      reinject: async () => {
+        throw new Error("reinject must not run after a timeout");
+      },
+    };
+    const recordOutcomeSpy = spyOn(loop.compactionCircuit, "recordOutcome");
+
+    // WHEN the compaction pipeline throws a PluginTimeoutError
+    const result = await loop.run([userMessage], () => {}, {
+      resolveContextWindow: () => ({
+        maxInputTokens: 10,
+        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
+      }),
+      compaction,
+      turnContext: timeoutCompactionTurnContext(),
+    });
+
+    // THEN the loop records the timeout as a compaction failure against its
+    // own circuit breaker, and yields for budget so the orchestrator can
+    // escalate.
+    expect(recordOutcomeSpy).toHaveBeenCalledTimes(1);
+    expect(recordOutcomeSpy.mock.calls[0]?.[1]).toBe(true);
+    expect(result.exitReason).toBe("budget");
+  });
+
+  test("yields 'budget' when inline compaction reports exhausted", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
+
+    const compaction: MidLoopCompaction = {
+      prepare: (history) => ({ rawHistory: history, options: undefined }),
+      applyResult: async () => {},
+      reinject: async () => {
+        throw new Error("reinject must not run when exhausted");
+      },
+    };
+
+    // WHEN compaction exhausts its retry budget
+    const result = await loop.run([userMessage], () => {}, {
+      resolveContextWindow: () => ({
+        maxInputTokens: 10,
+        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
+      }),
+      compaction,
+      turnContext: fakeCompactionTurnContext({
+        compacted: false,
+        exhausted: true,
+      }),
+    });
+
+    // THEN the loop yields for budget so the orchestrator can escalate.
+    expect(result.exitReason).toBe("budget");
+  });
+
   test("emits 'error' when provider throws an unhandled error", async () => {
     const provider: Provider = {
       name: "broken",
@@ -211,7 +501,9 @@ describe("AgentLoop exit-reason instrumentation", () => {
     const loop = new AgentLoop(provider, "system prompt");
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => { events.push(e); });
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
 
     expect(countExitEvents(events)).toBe(1);
     expect(lastExitEvent(events)?.reason).toBe("error");
@@ -229,16 +521,15 @@ describe("AgentLoop exit-reason instrumentation", () => {
       isError: false,
       yieldToUser: true,
     });
-    const loop = new AgentLoop(
-      provider,
-      "system",
-      {},
-      dummyTools,
-      toolExecutor,
-    );
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => { events.push(e); });
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
 
     expect(countExitEvents(events)).toBe(1);
   });
@@ -254,16 +545,19 @@ describe("AgentLoop exit-reason instrumentation", () => {
       controller.abort();
       return { content: "ok", isError: false };
     };
-    const loop = new AgentLoop(
-      provider,
-      "system",
-      {},
-      dummyTools,
-      toolExecutor,
-    );
+    const loop = new AgentLoop(provider, "system", {
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+    await loop.run(
+      [userMessage],
+      (e) => {
+        events.push(e);
+      },
+      { signal: controller.signal },
+    );
 
     expect(countExitEvents(events)).toBe(1);
     expect(lastExitEvent(events)?.reason).toBe("aborted_during_tools");

package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts

@@ -0,0 +1,141 @@
+/**
+ * Verifies that the per-turn `mutableLatestUserMessage` plumbed into
+ * `AgentLoop.run()` surfaces on every `SendMessageOptions.config` the loop
+ * emits — this is the memory-v3-live cache-anchor signal (the latest user
+ * message carries a volatile `<memory>` block, so the provider must anchor its
+ * long-TTL cache breakpoint on the most recent STABLE message instead).
+ *
+ * Default behavior (option unset) must remain unchanged — the field is omitted
+ * from `providerConfig` rather than carrying `undefined`/`false`, so the wire
+ * is byte-identical to today when the flag is off.
+ */
+
+import { describe, expect, mock, test } from "bun:test";
+
+import { makeMockLogger } from "./helpers/mock-logger.js";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+import { AgentLoop } from "../agent/loop.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "hi" }],
+};
+
+function textResponse(text: string): ProviderResponse {
+  return {
+    content: [{ type: "text", text }],
+    model: "mock-model",
+    usage: { inputTokens: 1, outputTokens: 1 },
+    stopReason: "end_turn",
+  };
+}
+
+function toolUseResponse(
+  id: string,
+  name: string,
+  input: Record<string, unknown>,
+): ProviderResponse {
+  return {
+    content: [{ type: "tool_use", id, name, input }],
+    model: "mock-model",
+    usage: { inputTokens: 1, outputTokens: 1 },
+    stopReason: "tool_use",
+  };
+}
+
+function makeRecordingProvider(responses: ProviderResponse[]): {
+  provider: Provider;
+  configs: () => Array<Record<string, unknown> | undefined>;
+} {
+  const configs: Array<Record<string, unknown> | undefined> = [];
+  let i = 0;
+  const provider: Provider = {
+    name: "mock",
+    async sendMessage(
+      _messages: Message[],
+      options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      configs.push(options?.config as Record<string, unknown> | undefined);
+      const response = responses[i] ?? responses[responses.length - 1];
+      i++;
+      return response;
+    },
+  };
+  return { provider, configs: () => configs };
+}
+
+describe("AgentLoop.run — mutableLatestUserMessage plumbing", () => {
+  test("forwards mutableLatestUserMessage=true to providerConfig on every LLM call (multi-turn)", async () => {
+    const { provider, configs } = makeRecordingProvider([
+      toolUseResponse("t1", "echo", { value: "first" }),
+      textResponse("done"),
+    ]);
+
+    const dummyTools: ToolDefinition[] = [
+      {
+        name: "echo",
+        description: "Echo back the input",
+        input_schema: {
+          type: "object",
+          properties: { value: { type: "string" } },
+        },
+      },
+    ];
+
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+
+    const loop = new AgentLoop(provider, "system", {
+      config: { maxTokens: 1024 },
+      tools: dummyTools,
+      toolExecutor: toolExecutor,
+    });
+
+    await loop.run([userMessage], () => {}, {
+      callSite: "mainAgent",
+      mutableLatestUserMessage: true,
+    });
+
+    // Two sends — initial + one tool round-trip. Every send carries the flag.
+    expect(configs()).toHaveLength(2);
+    for (const cfg of configs()) {
+      expect(cfg?.mutableLatestUserMessage).toBe(true);
+    }
+  });
+
+  test("omits mutableLatestUserMessage when unset (default behavior unchanged)", async () => {
+    const { provider, configs } = makeRecordingProvider([textResponse("hi")]);
+    const loop = new AgentLoop(provider, "system", {
+      config: { maxTokens: 1024 },
+    });
+
+    await loop.run([userMessage], () => {});
+
+    expect(configs()).toHaveLength(1);
+    expect("mutableLatestUserMessage" in (configs()[0] ?? {})).toBe(false);
+  });
+
+  test("omits mutableLatestUserMessage when explicitly false (flag-off byte-identity)", async () => {
+    const { provider, configs } = makeRecordingProvider([textResponse("hi")]);
+    const loop = new AgentLoop(provider, "system", {
+      config: { maxTokens: 1024 },
+    });
+
+    await loop.run([userMessage], () => {}, {
+      callSite: "mainAgent",
+      mutableLatestUserMessage: false,
+    });
+
+    expect("mutableLatestUserMessage" in (configs()[0] ?? {})).toBe(false);
+  });
+});