@vellumai/assistant 0.8.9 → 0.8.10-dev.202606092334.09948c8

package/src/__tests__/context-window-manager-overflow-rung.test.ts

@@ -0,0 +1,351 @@
+/**
+ * Coverage for the turn-scoped overflow-recovery state machine that lives on
+ * `ContextWindowManager.reduceOverflowOneRung`. The manager holds the reducer
+ * state across the successive rung calls of a single turn, derives the
+ * compaction target from its own preflight budget corrected by the provider's
+ * actual token count, and starts a fresh ladder once the turn boundary resets
+ * it. `reduceContextOverflow` is mocked so these tests pin the manager's own
+ * orchestration without exercising the full reduction ladder.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+function makeLoggerStub(): Record<string, unknown> {
+  const stub: Record<string, unknown> = {};
+  for (const m of [
+    "info",
+    "warn",
+    "error",
+    "debug",
+    "trace",
+    "fatal",
+    "silent",
+    "child",
+  ]) {
+    stub[m] = m === "child" ? () => makeLoggerStub() : () => {};
+  }
+  return stub;
+}
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeLoggerStub(),
+}));
+
+// ── Module-level mock state ───────────────────────────────────────────
+// The reducer call is captured so tests can assert which state and config the
+// manager fed in; the returned step is seeded per call so tests can drive the
+// ladder forward across rungs.
+
+interface ReducerState {
+  appliedTiers: string[];
+  injectionMode: string;
+  exhausted: boolean;
+}
+
+interface ReducerStepResult {
+  messages: unknown;
+  tier: string;
+  state: ReducerState;
+  estimatedTokens: number;
+}
+
+let reduceCalls: Array<{
+  messages: unknown;
+  config: Record<string, unknown>;
+  state: ReducerState | undefined;
+}> = [];
+let reduceSteps: ReducerStepResult[] = [];
+
+const estimateReturns: number[] = [];
+
+mock.module("../context/token-estimator.js", () => ({
+  estimatePromptTokens: (): number => {
+    const next = estimateReturns.shift();
+    if (next === undefined) {
+      throw new Error(
+        "estimatePromptTokens called more times than estimateReturns seeded",
+      );
+    }
+    return next;
+  },
+  // Deterministic per-tool budget so tests can assert the manager fed the
+  // turn's resolved tool set (not the constructor-time snapshot) into the
+  // overflow estimate.
+  estimateToolsTokens: (tools: unknown[]): number => tools.length * 1_000,
+}));
+
+mock.module(
+  "../plugins/defaults/compaction/context-overflow-reducer.js",
+  () => ({
+    createInitialReducerState: (): ReducerState => ({
+      appliedTiers: [],
+      injectionMode: "full",
+      exhausted: false,
+    }),
+    reduceContextOverflow: async (
+      messages: unknown,
+      config: Record<string, unknown>,
+      state: ReducerState | undefined,
+    ): Promise<ReducerStepResult> => {
+      reduceCalls.push({ messages, config, state });
+      const idx = reduceCalls.length - 1;
+      const step = reduceSteps[idx];
+      if (!step) {
+        throw new Error(
+          `Mock reducer called ${reduceCalls.length} time(s) but only ${reduceSteps.length} step(s) seeded`,
+        );
+      }
+      return step;
+    },
+  }),
+);
+
+import { ContextWindowManager } from "../plugins/defaults/compaction/window-manager.js";
+import type { Message, Provider, ToolDefinition } from "../providers/types.js";
+
+function makeProvider(): Provider {
+  return {
+    name: "mock-provider",
+    sendMessage: async () => ({
+      content: [],
+      model: "mock-model",
+      usage: { inputTokens: 0, outputTokens: 0 },
+      stopReason: "end_turn",
+    }),
+  };
+}
+
+function makeMessages(count: number): Message[] {
+  const messages: Message[] = [];
+  for (let i = 0; i < count; i++) {
+    messages.push({
+      role: i % 2 === 0 ? "user" : "assistant",
+      content: [{ type: "text", text: `msg ${i}` }],
+    });
+  }
+  return messages;
+}
+
+function makeConfig(maxAttempts: number = 3) {
+  return {
+    enabled: true,
+    maxInputTokens: 200_000,
+    targetBudgetRatio: 0.3,
+    compactThreshold: 0.8,
+    summaryBudgetRatio: 0.05,
+    overflowRecovery: {
+      enabled: true,
+      safetyMarginRatio: 0.05,
+      maxAttempts,
+      interactiveLatestTurnCompression: "summarize" as const,
+      nonInteractiveLatestTurnCompression: "summarize" as const,
+    },
+  };
+}
+
+function buildManager(maxAttempts: number = 3): ContextWindowManager {
+  return new ContextWindowManager({
+    provider: makeProvider(),
+    systemPrompt: "you are a test assistant",
+    config: makeConfig(maxAttempts),
+    conversationId: "conv-test",
+  });
+}
+
+function makeStep(
+  appliedTiers: string[],
+  exhausted: boolean = false,
+): ReducerStepResult {
+  return {
+    messages: makeMessages(2),
+    tier: appliedTiers[appliedTiers.length - 1] ?? "emergency_compaction",
+    state: { appliedTiers, injectionMode: "full", exhausted },
+    estimatedTokens: 100_000,
+  };
+}
+
+describe("ContextWindowManager.reduceOverflowOneRung", () => {
+  beforeEach(() => {
+    reduceCalls = [];
+    reduceSteps = [];
+    estimateReturns.length = 0;
+  });
+
+  test("holds reducer state across rungs within a turn", async () => {
+    // GIVEN a manager and a reducer that advances the ladder one rung per call
+    estimateReturns.push(240_000, 240_000);
+    reduceSteps = [
+      makeStep(["emergency_compaction"]),
+      makeStep(["emergency_compaction", "forced_compaction"]),
+    ];
+    const manager = buildManager();
+
+    // WHEN two rungs run within the same turn
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the first call seeds an empty ladder
+    expect(reduceCalls).toHaveLength(2);
+    expect(reduceCalls[0]?.state?.appliedTiers).toEqual([]);
+    // AND the second call carries the first rung's state forward
+    expect(reduceCalls[1]?.state?.appliedTiers).toEqual([
+      "emergency_compaction",
+    ]);
+  });
+
+  test("reuses the first rung's corrected target across the turn's later rungs", async () => {
+    // GIVEN the provider rejected at 480k while the estimator counted 240k, and
+    // only one estimate is seeded so a second estimate call would throw
+    estimateReturns.push(240_000);
+    reduceSteps = [
+      makeStep(["emergency_compaction"]),
+      makeStep(["emergency_compaction", "forced_compaction"]),
+    ];
+    const manager = buildManager();
+
+    // WHEN two rungs run within the same turn with the provider's actual count
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: 480_000,
+      allowAutoCompressLatestTurn: false,
+    });
+    await manager.reduceOverflowOneRung(makeMessages(4), {
+      actualTokens: 480_000,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the corrected target (190k preflight / 2x error ratio) is computed
+    // once and reused, rather than re-derived against the shrunk prompt
+    expect(reduceCalls[0]?.config.targetTokens).toBe(95_000);
+    expect(reduceCalls[1]?.config.targetTokens).toBe(95_000);
+    expect(reduceCalls[0]?.config.previousEstimatedInputTokens).toBe(240_000);
+    expect(reduceCalls[1]?.config.previousEstimatedInputTokens).toBe(240_000);
+  });
+
+  test("resetOverflowRecovery starts a fresh ladder", async () => {
+    // GIVEN a manager whose ladder has already advanced one rung
+    estimateReturns.push(240_000, 240_000);
+    reduceSteps = [
+      makeStep(["emergency_compaction"]),
+      makeStep(["emergency_compaction"]),
+    ];
+    const manager = buildManager();
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // WHEN the turn boundary resets recovery and a new rung runs
+    manager.resetOverflowRecovery();
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the post-reset call seeds an empty ladder again
+    expect(reduceCalls[1]?.state?.appliedTiers).toEqual([]);
+  });
+
+  test("corrected target lowers below preflight when the provider under-counted", async () => {
+    // GIVEN the provider rejected at 480k while the estimator counted 240k
+    estimateReturns.push(240_000);
+    reduceSteps = [makeStep(["emergency_compaction"])];
+    const manager = buildManager();
+
+    // WHEN a rung runs with the provider's actual token count
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: 480_000,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the target is the preflight budget (200k * 0.95 = 190k) divided by
+    // the 2x estimation-error ratio
+    expect(reduceCalls[0]?.config.targetTokens).toBe(95_000);
+  });
+
+  test("estimates against the turn's resolved tool set, not the constructor snapshot", async () => {
+    // GIVEN a manager whose live tool set (3 tools) differs from the stale
+    // constructor-time budget snapshot
+    estimateReturns.push(240_000);
+    reduceSteps = [makeStep(["emergency_compaction"])];
+    const resolvedTools = [
+      { name: "a" },
+      { name: "b" },
+      { name: "c" },
+    ] as unknown as ToolDefinition[];
+    const manager = new ContextWindowManager({
+      provider: makeProvider(),
+      systemPrompt: "you are a test assistant",
+      config: makeConfig(),
+      conversationId: "conv-test",
+      toolTokenBudget: 50,
+      resolveTools: () => resolvedTools,
+    });
+
+    // WHEN a rung runs
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the reducer is fed the resolved tool budget (3 tools * 1k) rather
+    // than the constructor snapshot (50)
+    expect(reduceCalls[0]?.config.toolTokenBudget).toBe(3_000);
+  });
+
+  test("target is the full preflight budget when the actual count is unknown", async () => {
+    // GIVEN no provider-reported actual token count
+    estimateReturns.push(240_000);
+    reduceSteps = [makeStep(["emergency_compaction"])];
+    const manager = buildManager();
+
+    // WHEN a rung runs
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the target is the uncorrected preflight budget (200k * 0.95)
+    expect(reduceCalls[0]?.config.targetTokens).toBe(190_000);
+  });
+
+  test("long histories widen the safety margin", async () => {
+    // GIVEN a history longer than the 50-message threshold
+    estimateReturns.push(240_000);
+    reduceSteps = [makeStep(["emergency_compaction"])];
+    const manager = buildManager();
+
+    // WHEN a rung runs over 60 messages with no actual-token correction
+    await manager.reduceOverflowOneRung(makeMessages(60), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: false,
+    });
+
+    // THEN the safety margin bumps to 0.15 → preflight budget 200k * 0.85
+    expect(reduceCalls[0]?.config.targetTokens).toBe(170_000);
+  });
+
+  test("forwards the policy auto-compress permission to the ladder", async () => {
+    // GIVEN the overflow policy permits the terminal auto-compress rung
+    estimateReturns.push(240_000);
+    reduceSteps = [makeStep(["emergency_compaction"])];
+    const manager = buildManager();
+
+    // WHEN a rung runs
+    await manager.reduceOverflowOneRung(makeMessages(10), {
+      actualTokens: null,
+      allowAutoCompressLatestTurn: true,
+      overrideProfile: "fast",
+    });
+
+    // THEN the manager passes the policy decision and profile through to the
+    // reducer config alongside the budget it owns
+    expect(reduceCalls[0]?.config.allowAutoCompressLatestTurn).toBe(true);
+    expect(reduceCalls[0]?.config.overrideProfile).toBe("fast");
+    expect(reduceCalls[0]?.config.maxMiddleTierAttempts).toBe(3);
+  });
+});

package/src/__tests__/conversation-abort-tool-results.test.ts

@@ -177,6 +177,7 @@ mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
     async maybeCompact() {
       return { compacted: false };
     }
+    resetOverflowRecovery() {}
   },
   createContextSummaryMessage: () => ({
     role: "user",

package/src/__tests__/conversation-agent-loop-inference-profile.test.ts

@@ -435,6 +435,17 @@ function makeCtx(
       updateConfig: () => {},
       shouldCompact: () => ({ needed: false, estimatedTokens: 0 }),
       maybeCompact: async () => ({ compacted: false }),
+      resetOverflowRecovery: () => {},
+      reduceOverflowOneRung: async (msgs: Message[]) => ({
+        messages: msgs,
+        tier: "forced_compaction",
+        state: {
+          appliedTiers: ["forced_compaction"],
+          injectionMode: "full",
+          exhausted: true,
+        },
+        estimatedTokens: 1000,
+      }),
     } as unknown as Conversation["contextWindowManager"],
     contextCompactedMessageCount: 0,
     contextCompactedAt: null,

package/src/__tests__/conversation-agent-loop-overflow.test.ts

@@ -151,39 +151,70 @@ mock.module("../context/token-estimator.js", () => ({
 let mockReducerStepFn:
   | ((msgs: Message[], cfg: unknown, state: unknown) => unknown)
   | null = null;
+const makeInitialReducerState = () => ({
+  appliedTiers: [] as string[],
+  injectionMode: "full" as const,
+  exhausted: false,
+});
+const runMockReducer = async (
+  msgs: Message[],
+  cfg: unknown,
+  state: unknown,
+) => {
+  if (mockReducerStepFn) return mockReducerStepFn(msgs, cfg, state);
+  return {
+    messages: msgs,
+    tier: "forced_compaction",
+    state: {
+      appliedTiers: [
+        "forced_compaction",
+        "tool_result_truncation",
+        "media_stubbing",
+        "injection_downgrade",
+      ],
+      injectionMode: "full",
+      exhausted: true,
+    },
+    estimatedTokens: 1000,
+  };
+};
 mock.module(
   "../plugins/defaults/compaction/context-overflow-reducer.js",
   () => ({
-    createInitialReducerState: () => ({
-      appliedTiers: [],
-      injectionMode: "full" as const,
-      exhausted: false,
-    }),
-    reduceContextOverflow: async (
-      msgs: Message[],
-      cfg: unknown,
-      state: unknown,
-    ) => {
-      if (mockReducerStepFn) return mockReducerStepFn(msgs, cfg, state);
-      return {
-        messages: msgs,
-        tier: "forced_compaction",
-        state: {
-          appliedTiers: [
-            "forced_compaction",
-            "tool_result_truncation",
-            "media_stubbing",
-            "injection_downgrade",
-          ],
-          injectionMode: "full",
-          exhausted: true,
-        },
-        estimatedTokens: 1000,
-      };
-    },
+    createInitialReducerState: makeInitialReducerState,
+    reduceContextOverflow: runMockReducer,
   }),
 );
 
+// Stand-in for `ContextWindowManager`'s turn-scoped overflow ladder. Threads
+// reducer state across a turn's rungs and delegates each rung to the mocked
+// reducer, mirroring `reduceOverflowOneRung` / `resetOverflowRecovery` so the
+// convergence driver exercises the same per-rung escalation the real manager
+// drives.
+function makeOverflowLadderStub(): {
+  resetOverflowRecovery: () => void;
+  reduceOverflowOneRung: (
+    msgs: Message[],
+    opts: unknown,
+    signal?: AbortSignal,
+  ) => Promise<unknown>;
+} {
+  let state: unknown;
+  return {
+    resetOverflowRecovery: () => {
+      state = undefined;
+    },
+    reduceOverflowOneRung: async (msgs: Message[], opts: unknown) => {
+      if (!state) state = makeInitialReducerState();
+      const step = (await runMockReducer(msgs, opts, state)) as {
+        state: unknown;
+      };
+      state = step.state;
+      return step;
+    },
+  };
+}
+
 // Policy: default to fail_gracefully
 let mockOverflowAction: string = "fail_gracefully";
 mock.module("../daemon/context-overflow-policy.js", () => ({
@@ -599,6 +630,16 @@ function makeCtx(
 
     ...ctxOverrides,
   } as unknown as Conversation;
+  // The convergence driver resolves the turn-scoped overflow ladder off the
+  // manager; give every fake manager the ladder methods unless a test supplied
+  // its own.
+  const manager = ctx.contextWindowManager as unknown as Record<
+    string,
+    unknown
+  >;
+  if (typeof manager.reduceOverflowOneRung !== "function") {
+    Object.assign(manager, makeOverflowLadderStub());
+  }
   fakeContextWindowManagers.set(conversationId, ctx.contextWindowManager);
   return ctx;
 }
@@ -1721,28 +1762,30 @@ describe("session-agent-loop overflow recovery (JARVIS-110)", () => {
       return 170_000;
     };
 
-    // Convergence reducer becomes exhausted on the second tier so the
-    // loop escalates from convergence to the action-resolution block.
+    // The reduction ladder applies a middle tier first, then escalates to its
+    // terminal auto-compress-latest-turn rung and reports exhaustion. The
+    // terminal rung is what produces a `budget_yield_unrecovered` outcome when
+    // its rerun still yields at the mid-loop checkpoint.
     let reducerCallCount = 0;
     mockReducerStepFn = (msgs: Message[]) => {
       reducerCallCount++;
-      const exhausted = reducerCallCount >= 2;
+      const terminal = reducerCallCount >= 2;
       return {
         messages: msgs,
-        tier: exhausted ? "tool_result_truncation" : "forced_compaction",
+        tier: terminal ? "auto_compress_latest_turn" : "forced_compaction",
         state: {
-          appliedTiers: exhausted
-            ? ["forced_compaction", "tool_result_truncation"]
+          appliedTiers: terminal
+            ? ["forced_compaction", "auto_compress_latest_turn"]
             : ["forced_compaction"],
           injectionMode: "full" as const,
-          exhausted,
+          exhausted: terminal,
         },
-        estimatedTokens: exhausted ? 60_000 : 80_000,
+        estimatedTokens: terminal ? 60_000 : 80_000,
       };
     };
 
-    // The overflow policy directs us into auto_compress_latest_turn so the
-    // emergency compaction + final agentLoop.run path executes.
+    // The overflow policy permits the terminal auto-compress-latest-turn rung,
+    // so the ladder runs it as its final step.
     mockOverflowAction = "auto_compress_latest_turn";
 
     // Every provider call returns a tool_use, so each loop run does a tool
@@ -1752,19 +1795,16 @@ describe("session-agent-loop overflow recovery (JARVIS-110)", () => {
       toolUseResponse("tu-1", "bash", { command: "ls" }),
     ]);
 
-    // Every forced `maybeCompact` is invoked through three distinct call
-    // sites, all owned by the loop's pre-call budget gate / recovery ladder:
-    //   1. First-call (turn-start) gate (`force: true`) — the loop now owns
-    //      the turn-start compaction. It succeeds, but the mocked estimate
-    //      stays above the mid-loop threshold, so the turn proceeds and the
-    //      mid-loop gate still trips on the next iteration.
+    // Forced `maybeCompact` is invoked through the loop's own pre-call budget
+    // gate at two call sites (the reduction-ladder rungs are mocked out, so
+    // their summary calls don't reach this manager):
+    //   1. First-call (turn-start) gate (`force: true`) — the loop owns the
+    //      turn-start compaction. It succeeds, but the mocked estimate stays
+    //      above the mid-loop threshold, so the turn proceeds and the mid-loop
+    //      gate still trips on the next iteration.
     //   2. Mid-loop after the first tool turn (`force: true`) — must signal
     //      `exhausted: true` so the daemon escalates to the convergence
     //      reducer instead of looping forever.
-    //   3. auto_compress_latest_turn emergency compaction (`force: true`,
-    //      `minKeepRecentUserTurns: 0`) — succeeds and drops tokens below
-    //      threshold; the subsequent rerun yields again and is classified
-    //      as BUDGET_YIELD_UNRECOVERED.
     let forcedMaybeCompactCallCount = 0;
     const ctx = makeCtx({
       loopProvider: provider,
@@ -1843,7 +1883,8 @@ describe("session-agent-loop overflow recovery (JARVIS-110)", () => {
               exhausted: true,
             };
           }
-          // Emergency compaction call from auto_compress_latest_turn.
+          // Defensive fallback for any additional forced probe; the two gate
+          // call sites above are the only ones this test exercises.
           return {
             compacted: true,
             messages: [
@@ -1853,7 +1894,7 @@ describe("session-agent-loop overflow recovery (JARVIS-110)", () => {
               },
             ] as Message[],
             compactedPersistedMessages: 5,
-            summaryText: "Emergency summary",
+            summaryText: "Fallback summary",
             previousEstimatedInputTokens: 170_000,
             estimatedInputTokens: 90_000,
             maxInputTokens: 200_000,