@vellumai/assistant 0.8.12 → 0.9.0-staging.2

package/src/__tests__/compaction-trail-store.test.ts

@@ -35,6 +35,7 @@ import { getDb } from "../memory/db-connection.js";
 import { initializeDb } from "../memory/db-init.js";
 import {
   getCompactionLogsBetween,
+  getRequestLogMetaById,
   recordRequestLog,
 } from "../memory/llm-request-log-store.js";
 import { llmRequestLogs } from "../memory/schema.js";
@@ -56,10 +57,11 @@ function insertLogAt(
   conversationId: string,
   createdAt: number,
   callSite: "mainAgent" | "compactionAgent" | null,
+  requestPayload = "{}",
 ): string {
   const id = recordRequestLog(
     conversationId,
-    "{}",
+    requestPayload,
     "{}",
     undefined,
     "anthropic",
@@ -183,4 +185,75 @@ describe("getCompactionLogsBetween", () => {
     const result = getCompactionLogsBetween("nonexistent-conv", null, 500);
     expect(result).toEqual([]);
   });
+
+  test("computes requestMessageCount in SQL without returning the request payload", () => {
+    // GIVEN compaction rows whose request payloads use each provider
+    // shape (Anthropic/OpenAI `messages`, Gemini `contents`, OpenAI
+    // Responses `input`), plus rows where the count can't be derived
+    const conv = createConversation("msg-count");
+    insertLogAt(
+      conv.id,
+      100,
+      "compactionAgent",
+      JSON.stringify({ messages: [{}, {}, {}] }),
+    );
+    insertLogAt(
+      conv.id,
+      200,
+      "compactionAgent",
+      JSON.stringify({ contents: [{}, {}] }),
+    );
+    insertLogAt(
+      conv.id,
+      300,
+      "compactionAgent",
+      JSON.stringify({ input: [{}] }),
+    );
+    insertLogAt(conv.id, 400, "compactionAgent", "not-json{{{");
+    insertLogAt(conv.id, 500, "compactionAgent", JSON.stringify({}));
+
+    // WHEN querying the trail window
+    const result = getCompactionLogsBetween(conv.id, null, 600);
+
+    // THEN each row carries the SQL-computed message count (null when
+    // the payload is malformed or has no known message array)
+    expect(result.map((r) => r.requestMessageCount)).toEqual([
+      3,
+      2,
+      1,
+      null,
+      null,
+    ]);
+    // AND the rows never include the request payload column
+    for (const row of result) {
+      expect("requestPayload" in row).toBe(false);
+    }
+  });
+});
+
+describe("getRequestLogMetaById", () => {
+  beforeEach(() => {
+    resetTables();
+  });
+
+  test("returns the row's metadata without either payload column", () => {
+    // GIVEN a stored log row
+    const conv = createConversation("meta-lookup");
+    const id = insertLogAt(conv.id, 100, "compactionAgent", '{"big":true}');
+
+    // WHEN looking it up by id
+    const row = getRequestLogMetaById(id);
+
+    // THEN the metadata comes back payload-free
+    expect(row).not.toBeNull();
+    expect(row!.id).toBe(id);
+    expect(row!.conversationId).toBe(conv.id);
+    expect(row!.createdAt).toBe(100);
+    expect("requestPayload" in row!).toBe(false);
+    expect("responsePayload" in row!).toBe(false);
+  });
+
+  test("returns null for an unknown id", () => {
+    expect(getRequestLogMetaById("nonexistent")).toBeNull();
+  });
 });

package/src/__tests__/compaction.benchmark.test.ts

@@ -4,21 +4,54 @@
  * Measures compaction cost with a mock provider:
  * - compaction latency under threshold pressure
  * - no-op fast path for below-threshold histories
- * - token reduction below target budget
+ * - token reduction below the low-watermark target budget
  * - single-pass summarization (exactly 1 call)
- * - severe pressure overriding cooldown
  */
 import { describe, expect, mock, test } from "bun:test";
 
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+// The compactor reads the conversation's image attachments from the DB to
+// build its manifest; with no images these return empty.
+mock.module("../memory/conversation-crud.js", () => ({
+  getMessages: () => [],
+}));
+mock.module("../memory/attachments-store.js", () => ({
+  getAttachmentMetadataForMessage: () => [],
+  getAttachmentContent: () => null,
+}));
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+}));
+
 import { DEFAULT_CONFIG } from "../config/defaults.js";
 import { estimatePromptTokens } from "../context/token-estimator.js";
 import { ContextWindowManager } from "../plugins/defaults/compaction/window-manager.js";
 import type { Message, Provider } from "../providers/types.js";
 
-mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
-}));
+// Synthetic per-turn timestamp so the model's tail_start resolves to a message
+// index. Turn 10 anchors a fat verbatim tail near the front, forcing the
+// deterministic forward-cut to advance to meet the low-watermark budget.
+function turnTimestamp(turn: number): string {
+  const hour = String(10 + Math.floor(turn / 60)).padStart(2, "0");
+  const minute = String(turn % 60).padStart(2, "0");
+  return `2026-05-21 (Thursday) ${hour}:${minute}:00 -05:00 (America/Chicago)`;
+}
+
+function compactionResponse(callIndex: number): string {
+  return `<compaction_result>
+<summary>
+Compacted summary, pass ${callIndex}. Goals preserved, constraints noted.
+</summary>
+<key_state>
+- Nothing critical pending.
+</key_state>
+<tail_start timestamp="${turnTimestamp(10)}" preview="[U10] User message with enough" />
+</compaction_result>`;
+}
 
 function makeSummaryProvider(counter: { calls: number }): Provider {
   return {
@@ -26,12 +59,7 @@ function makeSummaryProvider(counter: { calls: number }): Provider {
     async sendMessage() {
       counter.calls += 1;
       return {
-        content: [
-          {
-            type: "text",
-            text: `## Goals\n- Preserve state\n## Constraints\n- Keep PRs small\n## Decisions\n- Call ${counter.calls}`,
-          },
-        ],
+        content: [{ type: "text", text: compactionResponse(counter.calls) }],
         model: "mock-model",
         usage: { inputTokens: 420, outputTokens: 85 },
         stopReason: "end_turn",
@@ -48,7 +76,9 @@ function makeLongMessages(turns: number): Message[] {
       content: [
         {
           type: "text",
-          text: `[U${i}] User message with enough content to estimate tokens. Topic ${
+          text: `<turn_context>\ncurrent_time: ${turnTimestamp(
+            i,
+          )}\n</turn_context>\n[U${i}] User message with enough content to estimate tokens. Topic ${
             i % 9
           }.`,
         },
@@ -73,31 +103,35 @@ function makeConfig() {
   return {
     ...DEFAULT_CONFIG.llm.default.contextWindow,
     maxInputTokens: 6000,
-    targetBudgetRatio: 0.58,
+    targetBudgetRatio: 0.4,
     compactThreshold: 0.6,
     summaryBudgetRatio: 0.05,
   };
 }
 
+function makeManager(provider: Provider): ContextWindowManager {
+  return new ContextWindowManager({
+    provider,
+    systemPrompt: "system prompt",
+    config: makeConfig(),
+    conversationId: "conv-benchmark",
+  });
+}
+
 describe("Compaction benchmark", () => {
   test("compaction with mock provider completes under 500ms", async () => {
     const counter = { calls: 0 };
     const provider = makeSummaryProvider(counter);
     const config = makeConfig();
-    const manager = new ContextWindowManager({
-      provider,
-      systemPrompt: "system prompt",
-      config,
-    });
+    const manager = makeManager(provider);
 
     // 90 turns = 180 messages, well above 60% of 6000 = 3600 threshold
     const messages = makeLongMessages(90);
     const before = estimatePromptTokens(messages, "system prompt", {
       providerName: "mock",
     });
-    expect(before).toBeGreaterThan(
-      config.maxInputTokens * config.compactThreshold,
-    );
+    // Real loader's default compaction.autoThreshold is 0.7.
+    expect(before).toBeGreaterThan(config.maxInputTokens * 0.7);
 
     const start = performance.now();
     const result = await manager.maybeCompact(messages);
@@ -110,12 +144,7 @@ describe("Compaction benchmark", () => {
   test("below-threshold check returns in under 50ms (no-op)", async () => {
     const counter = { calls: 0 };
     const provider = makeSummaryProvider(counter);
-    const config = makeConfig();
-    const manager = new ContextWindowManager({
-      provider,
-      systemPrompt: "system prompt",
-      config,
-    });
+    const manager = makeManager(provider);
 
     // 3 turns = 6 messages, well below threshold
     const messages = makeLongMessages(3);
@@ -125,7 +154,7 @@ describe("Compaction benchmark", () => {
     const elapsed = performance.now() - start;
 
     expect(result.compacted).toBe(false);
-    expect(result.reason).toBe("below compaction threshold");
+    expect(result.reason).toBe("below auto threshold");
     expect(elapsed).toBeLessThan(50);
     expect(counter.calls).toBe(0);
   });
@@ -134,11 +163,7 @@ describe("Compaction benchmark", () => {
     const counter = { calls: 0 };
     const provider = makeSummaryProvider(counter);
     const config = makeConfig();
-    const manager = new ContextWindowManager({
-      provider,
-      systemPrompt: "system prompt",
-      config,
-    });
+    const manager = makeManager(provider);
 
     const messages = makeLongMessages(90);
     const result = await manager.maybeCompact(messages);
@@ -147,7 +172,9 @@ describe("Compaction benchmark", () => {
     expect(result.estimatedInputTokens).toBeLessThan(
       result.previousEstimatedInputTokens,
     );
-    // Target is maxInputTokens * (targetBudgetRatio - summaryBudgetRatio)
+    // The deterministic forward-cut drives the rebuilt history at or below the
+    // low-watermark budget: maxInputTokens * (targetBudgetRatio -
+    // summaryBudgetRatio).
     const targetTokens = Math.floor(
       config.maxInputTokens *
         (config.targetBudgetRatio - config.summaryBudgetRatio),
@@ -158,12 +185,7 @@ describe("Compaction benchmark", () => {
   test("single-pass summarization makes exactly 1 summary call", async () => {
     const counter = { calls: 0 };
     const provider = makeSummaryProvider(counter);
-    const config = makeConfig();
-    const manager = new ContextWindowManager({
-      provider,
-      systemPrompt: "system prompt",
-      config,
-    });
+    const manager = makeManager(provider);
 
     const messages = makeLongMessages(90);
     const result = await manager.maybeCompact(messages);

package/src/__tests__/compactor-low-watermark-cut.test.ts

@@ -0,0 +1,349 @@
+/**
+ * Deterministic low-watermark forward-cut enforcement in
+ * `runAssistantDrivenCompaction`.
+ *
+ * When the model keeps a fat verbatim tail, the compactor advances the cut
+ * forward (dropping more leading messages into the summarized region) until the
+ * rebuilt-history estimate fits `targetTokens` — while never cutting past the
+ * most-recent-complete-exchange floor and never orphaning tool_use/tool_result
+ * pairs.
+ */
+import { describe, expect, mock, test } from "bun:test";
+
+function makeLoggerStub(): Record<string, unknown> {
+  const stub: Record<string, unknown> = {};
+  for (const m of [
+    "info",
+    "warn",
+    "error",
+    "debug",
+    "trace",
+    "fatal",
+    "silent",
+    "child",
+  ]) {
+    stub[m] = m === "child" ? () => makeLoggerStub() : () => {};
+  }
+  return stub;
+}
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeLoggerStub(),
+}));
+
+mock.module("../memory/conversation-crud.js", () => ({
+  getMessages: () => [],
+}));
+
+mock.module("../memory/attachments-store.js", () => ({
+  getAttachmentMetadataForMessage: () => [],
+  getAttachmentContent: () => null,
+}));
+
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+}));
+
+import { runAssistantDrivenCompaction } from "../context/compactor.js";
+import { estimatePromptTokens } from "../context/token-estimator.js";
+import type { Message, Provider } from "../providers/types.js";
+
+function turnTimestamp(turn: number): string {
+  const hour = String(10 + Math.floor(turn / 60)).padStart(2, "0");
+  const minute = String(turn % 60).padStart(2, "0");
+  return `2026-05-21 (Thursday) ${hour}:${minute}:00 -05:00 (America/Chicago)`;
+}
+
+function userTurn(turn: number, body: string): Message {
+  return {
+    role: "user",
+    content: [
+      {
+        type: "text",
+        text: `<turn_context>\ncurrent_time: ${turnTimestamp(
+          turn,
+        )}\n</turn_context>\n[U${turn}] ${body}`,
+      },
+    ],
+  };
+}
+
+function assistantTurn(turn: number, body: string): Message {
+  return {
+    role: "assistant",
+    content: [{ type: "text", text: `[A${turn}] ${body}` }],
+  };
+}
+
+function makeProvider(response: string): Provider {
+  return {
+    name: "mock-provider",
+    sendMessage: async () => ({
+      content: [{ type: "text", text: response }],
+      model: "mock-model",
+      usage: { inputTokens: 100, outputTokens: 50 },
+      stopReason: "end_turn",
+    }),
+  };
+}
+
+const SUMMARY = "Earlier turns summarized in the assistant's own voice.";
+
+function compactionResponse(tailTurn: number, preview: string): string {
+  return `<compaction_result>
+<summary>
+${SUMMARY}
+</summary>
+<key_state>
+- Nothing critical pending.
+</key_state>
+<tail_start timestamp="${turnTimestamp(tailTurn)}" preview="${preview}" />
+</compaction_result>`;
+}
+
+describe("runAssistantDrivenCompaction — low-watermark forward cut", () => {
+  test("advances the model's tail forward until the rebuilt history fits the target", async () => {
+    // 40 turns = 80 messages, each user body is heavy so the verbatim tail is
+    // expensive. The model anchors its tail at turn 4 (keeping ~72 messages);
+    // the forward-cut must advance far past that to meet the target.
+    const messages: Message[] = [];
+    for (let i = 0; i < 40; i++) {
+      messages.push(
+        userTurn(
+          i,
+          "User message with a fairly long body so that each turn carries real token weight in the estimate. ".repeat(
+            3,
+          ),
+        ),
+      );
+      messages.push(
+        assistantTurn(
+          i,
+          "Assistant reply, also reasonably long to contribute estimate weight. ".repeat(
+            3,
+          ),
+        ),
+      );
+    }
+
+    const targetTokens = 1500;
+    const result = await runAssistantDrivenCompaction({
+      conversationId: "conv-test",
+      messages,
+      provider: makeProvider(
+        compactionResponse(4, "User message with a fairly long body"),
+      ),
+      systemPrompt: "system",
+      compaction: { enabled: true, autoThreshold: 0.7 },
+      maxInputTokens: 100_000,
+      targetTokens,
+      previousEstimatedInputTokens: 90_000,
+    });
+
+    expect(result.compacted).toBe(true);
+
+    // The rebuilt history (summary + verbatim tail) lands at or below target.
+    const estimate = estimatePromptTokens(result.messages, "system", {
+      providerName: "mock-provider",
+    });
+    expect(estimate).toBeLessThanOrEqual(targetTokens);
+
+    // The forward-cut advanced well past the model's chosen anchor (turn 4 →
+    // index 8): far fewer tail messages survive than the model kept.
+    expect(result.preservedTailMessages).toBeLessThan(messages.length - 8);
+    expect(result.preservedTailMessages).toBeGreaterThan(0);
+
+    // The cut found a tail that fits the budget, so the floor was not the
+    // binding constraint — no futile-retry signal.
+    expect(result.tailFloorReached).toBe(false);
+  });
+
+  test("never advances past the most-recent-complete-exchange floor", async () => {
+    // A small conversation where the target is unreachably tiny: the cut must
+    // still preserve at least the last complete user→assistant exchange.
+    const messages: Message[] = [];
+    for (let i = 0; i < 6; i++) {
+      messages.push(userTurn(i, "short user turn body here"));
+      messages.push(assistantTurn(i, "short assistant reply body here"));
+    }
+
+    const result = await runAssistantDrivenCompaction({
+      conversationId: "conv-test",
+      messages,
+      provider: makeProvider(
+        compactionResponse(1, "short user turn body here"),
+      ),
+      systemPrompt: "system",
+      compaction: { enabled: true, autoThreshold: 0.7 },
+      maxInputTokens: 100_000,
+      // Target far below anything achievable — forces the cut to the floor.
+      targetTokens: 1,
+      previousEstimatedInputTokens: 90_000,
+    });
+
+    expect(result.compacted).toBe(true);
+    // The last complete exchange (final user + final assistant) survives: the
+    // floor is the start of that exchange, so at least 2 tail messages remain.
+    expect(result.preservedTailMessages).toBeGreaterThanOrEqual(2);
+    // The tail still opens on a user turn (clean forward-cut boundary).
+    const tail = result.messages.slice(
+      result.messages.length - (result.preservedTailMessages ?? 0),
+    );
+    expect(tail[0]?.role).toBe("user");
+    // The cut advanced to the floor but the surviving tail is still over the
+    // (unreachably tiny) target — this is the floor-dominated case the
+    // window-manager uses to skip a futile retry.
+    expect(result.tailFloorReached).toBe(true);
+  });
+
+  test("forward cut lands on a clean user boundary, never orphaning a tool_result", async () => {
+    // Build: [u0,a0, u1,a1(tool_use X), u2(tool_result X), a2, u3,a3].
+    // The model anchors at turn 0; a naive forward cut could land on the
+    // tool_result-only user message and orphan tool_use X. The boundary check
+    // must skip it.
+    const toolUse: Message = {
+      role: "assistant",
+      content: [{ type: "tool_use", id: "X", name: "Bash", input: {} }],
+    };
+    const toolResult: Message = {
+      role: "user",
+      content: [{ type: "tool_result", tool_use_id: "X", content: "ok" }],
+    };
+    const messages: Message[] = [
+      userTurn(
+        0,
+        "first user turn with a long body to weigh the estimate. ".repeat(4),
+      ),
+      assistantTurn(0, "first assistant reply with a long body. ".repeat(4)),
+      userTurn(
+        1,
+        "second user turn, also long, to add estimate weight. ".repeat(4),
+      ),
+      toolUse,
+      toolResult,
+      assistantTurn(2, "assistant continues after the tool result. ".repeat(4)),
+      userTurn(3, "final user turn body"),
+      assistantTurn(3, "final assistant reply body"),
+    ];
+
+    const result = await runAssistantDrivenCompaction({
+      conversationId: "conv-test",
+      messages,
+      provider: makeProvider(
+        compactionResponse(1, "second user turn, also long"),
+      ),
+      systemPrompt: "system",
+      compaction: { enabled: true, autoThreshold: 0.7 },
+      maxInputTokens: 100_000,
+      // Small target to force forward advancement past the tool cluster.
+      targetTokens: 200,
+      previousEstimatedInputTokens: 90_000,
+    });
+
+    expect(result.compacted).toBe(true);
+    const tail = result.messages.slice(
+      result.messages.length - (result.preservedTailMessages ?? 0),
+    );
+    // The tail must not open with an orphaned tool_result.
+    const opensWithToolResult =
+      tail[0]?.role === "user" &&
+      tail[0].content.some((b) => b.type === "tool_result");
+    expect(opensWithToolResult).toBe(false);
+  });
+
+  test("keeps the model's tail untouched when it already fits the budget", async () => {
+    // 10 turns = 20 messages; the model anchors its tail at turn 4 (index 8)
+    // and the target is far above the rebuilt-history estimate. Enforcement
+    // must not advance the cut at all — the cut is enforcement, not
+    // optimization.
+    const messages: Message[] = [];
+    for (let i = 0; i < 10; i++) {
+      messages.push(userTurn(i, "short user turn body here"));
+      messages.push(assistantTurn(i, "short assistant reply body here"));
+    }
+
+    const result = await runAssistantDrivenCompaction({
+      conversationId: "conv-test",
+      messages,
+      provider: makeProvider(
+        compactionResponse(4, "short user turn body here"),
+      ),
+      systemPrompt: "system",
+      compaction: { enabled: true, autoThreshold: 0.7 },
+      maxInputTokens: 100_000,
+      targetTokens: 50_000,
+      previousEstimatedInputTokens: 90_000,
+    });
+
+    expect(result.compacted).toBe(true);
+    // The model's cut (turn 4 → index 8) keeps messages 8..19 = 12 messages.
+    expect(result.preservedTailMessages).toBe(12);
+    // No span was dropped, so the summary carries no truncation notice.
+    const summaryBlock = result.messages[0]?.content.find(
+      (b) => b.type === "text",
+    );
+    expect(
+      summaryBlock && "text" in summaryBlock ? summaryBlock.text : "",
+    ).not.toContain("Context budget enforcement");
+  });
+
+  test("acknowledges the span dropped by an advanced cut in the summary message", async () => {
+    // Same shape as the first test: a heavy conversation where the model's
+    // tail choice overshoots the budget and the cut must advance. The span
+    // between the model's cut and the enforced cut is in neither the
+    // summary's detail nor the retained tail, so the summary message must
+    // carry an explicit truncation notice.
+    const messages: Message[] = [];
+    for (let i = 0; i < 40; i++) {
+      messages.push(
+        userTurn(
+          i,
+          "User message with a fairly long body so that each turn carries real token weight in the estimate. ".repeat(
+            3,
+          ),
+        ),
+      );
+      messages.push(
+        assistantTurn(
+          i,
+          "Assistant reply, also reasonably long to contribute estimate weight. ".repeat(
+            3,
+          ),
+        ),
+      );
+    }
+
+    const result = await runAssistantDrivenCompaction({
+      conversationId: "conv-test",
+      messages,
+      provider: makeProvider(
+        compactionResponse(4, "User message with a fairly long body"),
+      ),
+      systemPrompt: "system",
+      compaction: { enabled: true, autoThreshold: 0.7 },
+      maxInputTokens: 100_000,
+      targetTokens: 1500,
+      previousEstimatedInputTokens: 90_000,
+    });
+
+    expect(result.compacted).toBe(true);
+    const summaryBlock = result.messages[0]?.content.find(
+      (b) => b.type === "text",
+    );
+    const summaryTextOut =
+      summaryBlock && "text" in summaryBlock ? summaryBlock.text : "";
+    // The notice names the dropped span (count by role) so the loss is
+    // visible in-context rather than silent.
+    expect(summaryTextOut).toContain("Context budget enforcement");
+    expect(summaryTextOut).toMatch(
+      /\d+ message\(s\) \(\d+ user, \d+ assistant\)/,
+    );
+    // The original summary body is preserved verbatim ahead of the notice.
+    expect(summaryTextOut).toContain(SUMMARY);
+    // The notice also rides the DURABLE summary — applyCompactionResult
+    // persists and rehydrates from `result.summaryText`, so a notice that
+    // lived only on the in-memory message would vanish on reload/fork.
+    expect(result.summaryText).toContain("Context budget enforcement");
+    expect(result.summaryText).toBe(summaryTextOut);
+  });
+});