@vellumai/assistant 0.8.8 → 0.8.9-dev.202606091853.fbaa2ae

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -13,17 +13,22 @@
  * Sites not exercised here (`aborted_via_error`) require deeper provider
  * fakery and are best covered by integration tests.
  */
-import { describe, expect, spyOn, test } from "bun:test";
+import { afterEach, describe, expect, mock, test } from "bun:test";
 
 import type {
   AgentEvent,
   CheckpointDecision,
   CheckpointInfo,
-  MidLoopCompaction,
 } from "../agent/loop.js";
 import { AgentLoop, isMaxTokensStopReason } from "../agent/loop.js";
-import type { TurnContext } from "../plugins/types.js";
-import { PluginTimeoutError } from "../plugins/types.js";
+import type { ContextWindowConfig } from "../config/types.js";
+import type { TrustContext } from "../daemon/trust-context.js";
+import {
+  createContextWindowManager,
+  disposeContextWindowManager,
+  getContextWindowManager,
+} from "../plugins/defaults/compaction/manager-store.js";
+import type { PostCompactContext } from "../plugins/defaults/memory-retrieval/hooks/post-compact.js";
 import type {
   Message,
   Provider,
@@ -32,6 +37,23 @@ import type {
   ToolDefinition,
 } from "../providers/types.js";
 
+// The agent loop invokes the default post-compaction re-injection hook directly
+// when it compacts in place. Stub it so these unit tests can drive the
+// re-injection result without the daemon-level injector chain. Tests assign
+// `postCompactImpl` to observe the call or force a failure; when unset
+// the hook is a no-op that returns the history it was handed.
+let postCompactImpl:
+  | ((input: PostCompactContext) => Promise<Message[]>)
+  | null = null;
+mock.module(
+  "../plugins/defaults/memory-retrieval/hooks/post-compact.js",
+  () => ({
+    default: async (input: PostCompactContext) => ({
+      messages: postCompactImpl ? await postCompactImpl(input) : input.history,
+    }),
+  }),
+);
+
 // ---------------------------------------------------------------------------
 // Helpers (mirrored from agent-loop.test.ts so this file is self-contained)
 // ---------------------------------------------------------------------------
@@ -101,38 +123,26 @@ const userMessage: Message = {
   content: [{ type: "text", text: "Hello" }],
 };
 
-// A turn context whose `contextWindowManager.maybeCompact` returns a canned
-// result, so the loop's native compaction pipeline runs without the real
-// orchestrator machinery.
-function fakeCompactionTurnContext(result: {
-  compacted: boolean;
-  exhausted: boolean;
-}): TurnContext {
-  return {
-    requestId: "req-compact",
-    conversationId: "conv-compact",
-    turnIndex: 0,
-    trust: { sourceChannel: "vellum", trustClass: "unknown" },
-    contextWindowManager: {
-      maybeCompact: async () => result,
-    },
-  } as unknown as TurnContext;
-}
-
-// A turn context whose compaction call times out, exercising the loop's
-// PluginTimeoutError handling.
-function timeoutCompactionTurnContext(): TurnContext {
-  return {
-    requestId: "req-compact",
-    conversationId: "conv-compact",
-    turnIndex: 0,
-    trust: { sourceChannel: "vellum", trustClass: "unknown" },
-    contextWindowManager: {
-      maybeCompact: async () => {
-        throw new PluginTimeoutError("compaction", "default-compaction", 1);
-      },
-    },
-  } as unknown as TurnContext;
+// Register a context-window manager whose `maybeCompact` returns a canned
+// result in the compaction store the loop resolves from, so the loop's
+// compaction call runs without the real orchestrator machinery. Returns the
+// trust snapshot the loop also needs as a run option.
+function fakeCompaction(
+  conversationId: string,
+  result: { compacted: boolean; exhausted: boolean },
+): { trust: TrustContext } {
+  createContextWindowManager({
+    provider: { name: "mock-provider" } as unknown as Provider,
+    systemPrompt: "system",
+    config: {} as unknown as ContextWindowConfig,
+    conversationId,
+  });
+  const manager = getContextWindowManager(conversationId);
+  if (manager) {
+    manager.maybeCompact = (async () =>
+      result) as unknown as typeof manager.maybeCompact;
+  }
+  return { trust: { sourceChannel: "vellum", trustClass: "unknown" } };
 }
 
 function lastExitEvent(
@@ -153,6 +163,10 @@ function countExitEvents(events: AgentEvent[]): number {
 // ---------------------------------------------------------------------------
 
 describe("AgentLoop exit-reason instrumentation", () => {
+  afterEach(() => {
+    disposeContextWindowManager("test-conversation");
+  });
+
   test("recognizes provider output-token stop reasons", () => {
     expect(isMaxTokensStopReason("max_tokens")).toBe(true);
     expect(isMaxTokensStopReason("MAX_TOKENS")).toBe(true);
@@ -164,11 +178,19 @@ describe("AgentLoop exit-reason instrumentation", () => {
 
   test("emits exit event exactly once with 'no_tool_calls' on plain text response", async () => {
     const { provider } = createMockProvider([textResponse("Hi there!")]);
-    const loop = new AgentLoop(provider, "system prompt");
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => {
-      events.push(e);
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(countExitEvents(events)).toBe(1);
@@ -178,11 +200,19 @@ describe("AgentLoop exit-reason instrumentation", () => {
 
   test("agent_loop_exit is the last event emitted", async () => {
     const { provider } = createMockProvider([textResponse("Hi there!")]);
-    const loop = new AgentLoop(provider, "system prompt");
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => {
-      events.push(e);
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(events.length).toBeGreaterThan(0);
@@ -193,11 +223,19 @@ describe("AgentLoop exit-reason instrumentation", () => {
     const { provider } = createMockProvider([
       maxTokensResponse("Partial answer"),
     ]);
-    const loop = new AgentLoop(provider, "system prompt");
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => {
-      events.push(e);
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(events.map((e) => e.type)).toEqual([
@@ -228,13 +266,20 @@ describe("AgentLoop exit-reason instrumentation", () => {
         stopReason: "max_tokens",
       },
     ]);
-    const loop = new AgentLoop(provider, "system prompt", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
       tools: dummyTools,
     });
 
     const events: AgentEvent[] = [];
-    const { history: result } = await loop.run([userMessage], (e) => {
-      events.push(e);
+    const { history: result } = await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(events.some((e) => e.type === "tool_use")).toBe(false);
@@ -246,19 +291,24 @@ describe("AgentLoop exit-reason instrumentation", () => {
 
   test("emits 'aborted_pre_call' when signal is already aborted at run start", async () => {
     const { provider } = createMockProvider([textResponse("never sent")]);
-    const loop = new AgentLoop(provider, "system prompt");
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
+    });
 
     const controller = new AbortController();
     controller.abort();
 
     const events: AgentEvent[] = [];
-    await loop.run(
-      [userMessage],
-      (e) => {
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
         events.push(e);
       },
-      { signal: controller.signal },
-    );
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      signal: controller.signal,
+    });
 
     expect(countExitEvents(events)).toBe(1);
     expect(lastExitEvent(events)?.reason).toBe("aborted_pre_call");
@@ -273,14 +323,21 @@ describe("AgentLoop exit-reason instrumentation", () => {
       isError: false,
       yieldToUser: true,
     });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => {
-      events.push(e);
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(countExitEvents(events)).toBe(1);
@@ -293,7 +350,10 @@ describe("AgentLoop exit-reason instrumentation", () => {
       textResponse("never reached"),
     ]);
     const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
@@ -302,13 +362,14 @@ describe("AgentLoop exit-reason instrumentation", () => {
       "budget";
 
     const events: AgentEvent[] = [];
-    await loop.run(
-      [userMessage],
-      (e) => {
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
         events.push(e);
       },
-      { onCheckpoint },
-    );
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      onCheckpoint,
+    });
 
     expect(countExitEvents(events)).toBe(0);
   });
@@ -321,7 +382,10 @@ describe("AgentLoop exit-reason instrumentation", () => {
       textResponse("never reached"),
     ]);
     const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
@@ -329,7 +393,10 @@ describe("AgentLoop exit-reason instrumentation", () => {
     // AND an effective context window so small that any real token estimate
     // of the running history exceeds the mid-loop threshold.
     // WHEN the loop checkpoints after the tool results land
-    const result = await loop.run([userMessage], () => {}, {
+    const result = await loop.run({
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
       resolveContextWindow: () => ({
         maxInputTokens: 10,
         overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
@@ -348,13 +415,19 @@ describe("AgentLoop exit-reason instrumentation", () => {
       textResponse("done"),
     ]);
     const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
 
     // WHEN the loop runs to completion
-    const result = await loop.run([userMessage], () => {}, {
+    const result = await loop.run({
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
       resolveContextWindow: () => ({
         maxInputTokens: 10,
         overflowRecovery: { enabled: false, safetyMarginRatio: 0 },
@@ -373,38 +446,37 @@ describe("AgentLoop exit-reason instrumentation", () => {
       textResponse("done after compaction"),
     ]);
     const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
 
     let reinjected = false;
     const events: AgentEvent[] = [];
-    const compaction: MidLoopCompaction = {
-      postCompactionHook: async () => {
-        reinjected = true;
-        return [userMessage];
-      },
+    postCompactImpl = async () => {
+      reinjected = true;
+      return [userMessage];
     };
 
     // WHEN the in-loop budget gate trips at the checkpoint
-    const result = await loop.run(
-      [userMessage],
-      (event) => {
+    const result = await loop.run({
+      messages: [userMessage],
+      onEvent: (event) => {
         events.push(event);
       },
-      {
-        resolveContextWindow: () => ({
-          maxInputTokens: 10,
-          overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
-        }),
-        compaction,
-        turnContext: fakeCompactionTurnContext({
-          compacted: true,
-          exhausted: false,
-        }),
-      },
-    );
+      resolveContextWindow: () => ({
+        maxInputTokens: 10,
+        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
+      }),
+      compactInPlace: true,
+      ...fakeCompaction("test-conversation", {
+        compacted: true,
+        exhausted: false,
+      }),
+    });
 
     // THEN the loop runs the compaction ceremony in place and continues to a
     // clean exit instead of yielding for budget. The durable commit is
@@ -416,67 +488,36 @@ describe("AgentLoop exit-reason instrumentation", () => {
     expect(result.exitReason).not.toBe("budget");
   });
 
-  test("yields 'budget' when inline compaction times out", async () => {
-    const { provider } = createMockProvider([
-      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
-      textResponse("never reached"),
-    ]);
-    const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
-      tools: dummyTools,
-      toolExecutor: toolExecutor,
-    });
-
-    const compaction: MidLoopCompaction = {
-      postCompactionHook: async () => {
-        throw new Error("postCompactionHook must not run after a timeout");
-      },
-    };
-    const recordOutcomeSpy = spyOn(loop.compactionCircuit, "recordOutcome");
-
-    // WHEN the compaction pipeline throws a PluginTimeoutError
-    const result = await loop.run([userMessage], () => {}, {
-      resolveContextWindow: () => ({
-        maxInputTokens: 10,
-        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
-      }),
-      compaction,
-      turnContext: timeoutCompactionTurnContext(),
-    });
-
-    // THEN the loop records the timeout as a compaction failure against its
-    // own circuit breaker, and yields for budget so the orchestrator can
-    // escalate.
-    expect(recordOutcomeSpy).toHaveBeenCalledTimes(1);
-    expect(recordOutcomeSpy.mock.calls[0]?.[1]).toBe(true);
-    expect(result.exitReason).toBe("budget");
-  });
-
   test("yields 'budget' when inline compaction reports exhausted", async () => {
     const { provider } = createMockProvider([
       toolUseResponse("t1", "read_file", { path: "/a.txt" }),
       textResponse("never reached"),
     ]);
     const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
 
-    const compaction: MidLoopCompaction = {
-      postCompactionHook: async () => {
-        throw new Error("postCompactionHook must not run when exhausted");
-      },
+    postCompactImpl = async () => {
+      throw new Error(
+        "post-compaction re-injection must not run when exhausted",
+      );
     };
 
     // WHEN compaction exhausts its retry budget
-    const result = await loop.run([userMessage], () => {}, {
+    const result = await loop.run({
+      messages: [userMessage],
+      onEvent: () => {},
       resolveContextWindow: () => ({
         maxInputTokens: 10,
         overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
       }),
-      compaction,
-      turnContext: fakeCompactionTurnContext({
+      compactInPlace: true,
+      ...fakeCompaction("test-conversation", {
         compacted: false,
         exhausted: true,
       }),
@@ -493,11 +534,19 @@ describe("AgentLoop exit-reason instrumentation", () => {
         throw new Error("provider exploded");
       },
     };
-    const loop = new AgentLoop(provider, "system prompt");
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
+    });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => {
-      events.push(e);
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(countExitEvents(events)).toBe(1);
@@ -516,14 +565,21 @@ describe("AgentLoop exit-reason instrumentation", () => {
       isError: false,
       yieldToUser: true,
     });
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
 
     const events: AgentEvent[] = [];
-    await loop.run([userMessage], (e) => {
-      events.push(e);
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
     });
 
     expect(countExitEvents(events)).toBe(1);
@@ -540,19 +596,23 @@ describe("AgentLoop exit-reason instrumentation", () => {
       controller.abort();
       return { content: "ok", isError: false };
     };
-    const loop = new AgentLoop(provider, "system", {
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
       tools: dummyTools,
       toolExecutor: toolExecutor,
     });
 
     const events: AgentEvent[] = [];
-    await loop.run(
-      [userMessage],
-      (e) => {
+    await loop.run({
+      messages: [userMessage],
+      onEvent: (e) => {
         events.push(e);
       },
-      { signal: controller.signal },
-    );
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      signal: controller.signal,
+    });
 
     expect(countExitEvents(events)).toBe(1);
     expect(lastExitEvent(events)?.reason).toBe("aborted_during_tools");