@vellumai/assistant 0.8.3 → 0.8.5

package/src/__tests__/list-messages-tool-merge.test.ts

@@ -91,7 +91,7 @@ describe("handleListMessages tool_result merging", () => {
       ]),
     );
 
-    const response = handleListMessages(createTestArgs(conv.id), null);
+    const response = handleListMessages(createTestArgs(conv.id));
     const body = response as { messages: MessagePayload[] };
 
     // Should be 2 messages: user prompt + assistant (tool_result user msg suppressed)
@@ -137,7 +137,7 @@ describe("handleListMessages tool_result merging", () => {
       ]),
     );
 
-    const response = handleListMessages(createTestArgs(conv.id), null);
+    const response = handleListMessages(createTestArgs(conv.id));
     const body = response as { messages: MessagePayload[] };
 
     expect(body.messages).toHaveLength(2);
@@ -167,7 +167,7 @@ describe("handleListMessages tool_result merging", () => {
       JSON.stringify([{ type: "text", text: "how are you?" }]),
     );
 
-    const response = handleListMessages(createTestArgs(conv.id), null);
+    const response = handleListMessages(createTestArgs(conv.id));
     const body = response as { messages: MessagePayload[] };
 
     expect(body.messages).toHaveLength(3);
@@ -175,11 +175,13 @@ describe("handleListMessages tool_result merging", () => {
     expect(body.messages[2].content).toBe("how are you?");
   });
 
-  test("tool_result at start of array (no preceding assistant) is preserved", async () => {
+  test("orphan tool_result at pagination boundary is suppressed entirely", async () => {
     const conv = createConversation();
-    // Orphan tool_result with no preceding assistant (pagination boundary).
-    // The preceding assistant tool_use lives in the previous page — dropping
-    // the result would be unrecoverable, so it is kept as-is.
+    // Orphan tool_result with no preceding assistant in this page. The
+    // matching tool_use lives on the previous page, so renderHistoryContent
+    // would silently drop the orphan downstream and leave a blank user
+    // bubble. The merger now suppresses the user row outright at the
+    // boundary to prevent that blank bubble from reaching the client.
     await addMessage(
       conv.id,
       "user",
@@ -197,19 +199,77 @@ describe("handleListMessages tool_result merging", () => {
       JSON.stringify([{ type: "text", text: "response" }]),
     );
 
-    const response = handleListMessages(createTestArgs(conv.id), null);
+    const response = handleListMessages(createTestArgs(conv.id));
+    const body = response as { messages: MessagePayload[] };
+
+    // User row dropped entirely; only the assistant survives.
+    expect(body.messages).toHaveLength(1);
+    expect(body.messages[0].role).toBe("assistant");
+    expect(body.messages[0].content).toBe("response");
+  });
+
+  test("mixed content at pagination boundary keeps real user text", async () => {
+    const conv = createConversation();
+    // Row has both real user text AND an orphan tool_result. The orphan
+    // gets stripped (it'd be dropped by renderHistoryContent anyway), but
+    // the user text is preserved.
+    await addMessage(
+      conv.id,
+      "user",
+      JSON.stringify([
+        {
+          type: "tool_result",
+          tool_use_id: "tu_orphan",
+          content: "stale result",
+        },
+        { type: "text", text: "what about this?" },
+      ]),
+    );
+    await addMessage(
+      conv.id,
+      "assistant",
+      JSON.stringify([{ type: "text", text: "answering" }]),
+    );
+
+    const response = handleListMessages(createTestArgs(conv.id));
     const body = response as { messages: MessagePayload[] };
 
-    // Orphan tool_result is preserved (not suppressed) to avoid data loss
     expect(body.messages).toHaveLength(2);
     expect(body.messages[0].role).toBe("user");
-    // The preserved message must retain the actual tool_result payload
-    const orphanToolCalls = body.messages[0].toolCalls;
-    expect(orphanToolCalls).toBeDefined();
-    expect(orphanToolCalls).toHaveLength(1);
-    expect(orphanToolCalls![0].result).toBe("stale result");
+    expect(body.messages[0].content).toBe("what about this?");
+    expect(body.messages[0].toolCalls).toBeUndefined();
     expect(body.messages[1].role).toBe("assistant");
-    expect(body.messages[1].content).toBe("response");
+    expect(body.messages[1].content).toBe("answering");
+  });
+
+  test("orphan tool_result + system_notice at boundary is suppressed", async () => {
+    const conv = createConversation();
+    // System notices ride alongside tool_results in the agent loop. At a
+    // boundary they shouldn't keep the row alive on their own — the row
+    // is still semantically tool-result-only from the user's perspective.
+    await addMessage(
+      conv.id,
+      "user",
+      JSON.stringify([
+        {
+          type: "tool_result",
+          tool_use_id: "tu_orphan",
+          content: "stale",
+        },
+        { type: "text", text: "<system_notice>internal</system_notice>" },
+      ]),
+    );
+    await addMessage(
+      conv.id,
+      "assistant",
+      JSON.stringify([{ type: "text", text: "ok" }]),
+    );
+
+    const response = handleListMessages(createTestArgs(conv.id));
+    const body = response as { messages: MessagePayload[] };
+
+    expect(body.messages).toHaveLength(1);
+    expect(body.messages[0].role).toBe("assistant");
   });
 
   test("multi-turn: each tool_result merges into correct assistant", async () => {
@@ -262,7 +322,7 @@ describe("handleListMessages tool_result merging", () => {
       JSON.stringify([{ type: "text", text: "thanks" }]),
     );
 
-    const response = handleListMessages(createTestArgs(conv.id), null);
+    const response = handleListMessages(createTestArgs(conv.id));
     const body = response as { messages: MessagePayload[] };
 
     // Consecutive assistant messages are merged at query time so the client
@@ -312,7 +372,7 @@ describe("handleListMessages tool_result merging", () => {
       ]),
     );
 
-    const response = handleListMessages(createTestArgs(conv.id), null);
+    const response = handleListMessages(createTestArgs(conv.id));
     const body = response as { messages: MessagePayload[] };
 
     expect(body.messages).toHaveLength(2);

package/src/__tests__/llm-context-normalization.test.ts

@@ -1152,7 +1152,6 @@ describe("normalizeLlmContextPayloads", () => {
         reasoning: { effort: "high" },
         max_output_tokens: 64000,
         stream: true,
-        store: false,
       },
       responsePayload: {
         id: "resp_abc123",
@@ -1245,7 +1244,6 @@ describe("normalizeLlmContextPayloads", () => {
           reasoning: { effort: "high" },
           max_output_tokens: 64000,
           stream: true,
-          store: false,
         },
         language: "json",
       },

package/src/__tests__/llm-request-log-call-site.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Tests for the `call_site` column on `llm_request_logs`. The column is
+ * stamped by `recordRequestLog` at insertion time and surfaces in
+ * `LogRow.callSite`. Historical rows (pre-migration 264) stay NULL —
+ * "we don't know" rather than guessing `mainAgent`.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    ui: {},
+    model: "test",
+    provider: "test",
+    memory: { enabled: false },
+    rateLimit: { maxRequestsPerMinute: 0 },
+    secretDetection: { enabled: false },
+  }),
+}));
+
+import { getDb, getSqliteFrom } from "../memory/db-connection.js";
+import { initializeDb } from "../memory/db-init.js";
+import {
+  getRequestLogById,
+  recordRequestLog,
+} from "../memory/llm-request-log-store.js";
+import { migrateLlmRequestLogCallSite } from "../memory/migrations/264-llm-request-log-call-site.js";
+import { llmRequestLogs } from "../memory/schema.js";
+
+initializeDb();
+
+function resetLogs(): void {
+  const db = getDb();
+  db.delete(llmRequestLogs).run();
+}
+
+describe("recordRequestLog call_site stamping", () => {
+  beforeEach(resetLogs);
+
+  test("stamps callSite when provided", () => {
+    const id = recordRequestLog(
+      "conv-1",
+      '{"req":1}',
+      '{"res":1}',
+      undefined,
+      "anthropic",
+      "mainAgent",
+    );
+    const row = getRequestLogById(id);
+    expect(row).not.toBeNull();
+    expect(row!.callSite).toBe("mainAgent");
+  });
+
+  test("leaves callSite NULL when omitted (backward compat)", () => {
+    const id = recordRequestLog("conv-1", '{"req":1}', '{"res":1}');
+    const row = getRequestLogById(id);
+    expect(row).not.toBeNull();
+    expect(row!.callSite).toBeNull();
+  });
+
+  test("supports the compactionAgent value", () => {
+    const id = recordRequestLog(
+      "conv-1",
+      '{"req":1}',
+      '{"res":1}',
+      undefined,
+      "anthropic",
+      "compactionAgent",
+    );
+    expect(getRequestLogById(id)?.callSite).toBe("compactionAgent");
+  });
+
+  test("two rows in the same conversation can carry different callSites", () => {
+    const mainId = recordRequestLog(
+      "conv-1",
+      '{"req":1}',
+      '{"res":1}',
+      undefined,
+      "anthropic",
+      "mainAgent",
+    );
+    const compactId = recordRequestLog(
+      "conv-1",
+      '{"req":2}',
+      '{"res":2}',
+      undefined,
+      "anthropic",
+      "compactionAgent",
+    );
+    expect(getRequestLogById(mainId)?.callSite).toBe("mainAgent");
+    expect(getRequestLogById(compactId)?.callSite).toBe("compactionAgent");
+  });
+});
+
+describe("migrateLlmRequestLogCallSite", () => {
+  test("adds the call_site column when missing", () => {
+    const db = getDb();
+    const raw = getSqliteFrom(db);
+
+    // Drop the column if present (simulate pre-264 state). SQLite supports
+    // `DROP COLUMN` since 3.35 (June 2021) — bun-sqlite ships well past that.
+    const before = raw
+      .query(`PRAGMA table_info(llm_request_logs)`)
+      .all() as Array<{ name: string }>;
+    if (before.some((c) => c.name === "call_site")) {
+      raw.exec(`ALTER TABLE llm_request_logs DROP COLUMN call_site`);
+    }
+
+    const without = raw
+      .query(`PRAGMA table_info(llm_request_logs)`)
+      .all() as Array<{ name: string }>;
+    expect(without.some((c) => c.name === "call_site")).toBe(false);
+
+    migrateLlmRequestLogCallSite(db);
+
+    const after = raw
+      .query(`PRAGMA table_info(llm_request_logs)`)
+      .all() as Array<{ name: string }>;
+    expect(after.some((c) => c.name === "call_site")).toBe(true);
+  });
+
+  test("is idempotent — second run is a no-op", () => {
+    const db = getDb();
+    // First run (column may or may not exist depending on test order; either
+    // path is fine for the idempotency contract).
+    migrateLlmRequestLogCallSite(db);
+    // Second run must not throw.
+    expect(() => migrateLlmRequestLogCallSite(db)).not.toThrow();
+  });
+});

package/src/__tests__/llm-request-log-source-clickhouse.test.ts

@@ -84,6 +84,7 @@ const SAMPLE_ROW = {
   // ClickHouse emits Int64 as a quoted string under JSONEachRow by default.
   created_at: "1778465138786",
   agent_loop_exit_reason: "no_tool_calls",
+  call_site: "mainAgent",
 };
 
 describe("ClickHouseLlmRequestLogSource", () => {
@@ -104,6 +105,7 @@ describe("ClickHouseLlmRequestLogSource", () => {
       responsePayload: '{"bar":2}',
       createdAt: 1778465138786,
       agentLoopExitReason: "no_tool_calls",
+      callSite: "mainAgent",
     });
   });
 
@@ -164,6 +166,30 @@ describe("ClickHouseLlmRequestLogSource", () => {
     expect(rows).toEqual([]);
   });
 
+  test("getRequestLogsByConversationId binds conversation id via parameterized placeholder", async () => {
+    // Regression guard: the conversation id flows from user-controlled
+    // input (conversationKey → conversations.id), so it must travel as a
+    // typed parameter, never as an inline string literal.
+    const recorder: FakeFetchCall[] = [];
+    const malicious = "conv-1' OR 1=1 --";
+    const src = makeSource({
+      body: JSON.stringify(SAMPLE_ROW) + "\n",
+      recorder,
+    });
+    const rows = await src.getRequestLogsByConversationId(malicious);
+    expect(rows).toHaveLength(1);
+    expect(recorder).toHaveLength(1);
+    const call = recorder[0]!;
+    const parsed = new URL(call.url);
+    expect(parsed.searchParams.get("param_assistant_id")).toBe(
+      "asst-fixture-001",
+    );
+    expect(parsed.searchParams.get("param_conversation_id")).toBe(malicious);
+    const body = String(call.init?.body ?? "");
+    expect(body).toContain("conversation_id = {conversation_id:String}");
+    expect(body).not.toContain(`'${malicious}'`);
+  });
+
   test("getRequestLogsByMessageId binds message ids via parameterized placeholders", async () => {
     // Regression for ATL-537. `getAssistantMessageIdsInTurn` returns the
     // caller-supplied id straight through when the message lookup misses,

package/src/__tests__/llm-resolver.test.ts

@@ -2,7 +2,10 @@ import { describe, expect, test } from "bun:test";
 
 import { z } from "zod";
 
-import { resolveCallSiteConfig } from "../config/llm-resolver.js";
+import {
+  resolveCallSiteConfig,
+  resolveDefaultProfileKey,
+} from "../config/llm-resolver.js";
 import { type LLMCallSite, LLMSchema } from "../config/schemas/llm.js";
 
 const fullDefault = {
@@ -690,13 +693,28 @@ describe("resolveCallSiteConfig", () => {
     });
 
     const callSites: LLMCallSite[] = [
-      "mainAgent", "subagentSpawn", "heartbeatAgent", "filingAgent",
-      "compactionAgent", "analyzeConversation", "callAgent",
-      "memoryExtraction", "memoryConsolidation", "memoryRetrieval",
-      "memoryRouter", "recall", "conversationSummarization",
-      "commitMessage", "conversationStarters", "replySuggestion",
-      "conversationTitle", "identityIntro", "emptyStateGreeting",
-      "notificationDecision", "interactionClassifier", "inference",
+      "mainAgent",
+      "subagentSpawn",
+      "heartbeatAgent",
+      "filingAgent",
+      "compactionAgent",
+      "analyzeConversation",
+      "callAgent",
+      "memoryExtraction",
+      "memoryConsolidation",
+      "memoryRetrieval",
+      "memoryRouter",
+      "recall",
+      "conversationSummarization",
+      "commitMessage",
+      "conversationStarters",
+      "replySuggestion",
+      "conversationTitle",
+      "identityIntro",
+      "emptyStateGreeting",
+      "notificationDecision",
+      "interactionClassifier",
+      "inference",
     ];
 
     for (const cs of callSites) {
@@ -778,7 +796,10 @@ describe("resolveCallSiteConfig", () => {
         provider_connection: "anthropic-managed",
       },
       profiles: {
-        fireworks: { provider: "fireworks", model: "accounts/fireworks/models/kimi-k2p5" },
+        fireworks: {
+          provider: "fireworks",
+          model: "accounts/fireworks/models/kimi-k2p5",
+        },
       },
       activeProfile: "fireworks",
     });
@@ -790,3 +811,134 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.provider_connection).toBe("anthropic-managed");
   });
 });
+
+describe("resolveDefaultProfileKey", () => {
+  test("mainAgent returns activeProfile when set and enabled", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+        gemini: { provider: "gemini", model: "gemini-2.5-pro" },
+      },
+      activeProfile: "gemini",
+    });
+    expect(resolveDefaultProfileKey("mainAgent", llm)).toBe("gemini");
+  });
+
+  test("mainAgent falls back to catalog default when activeProfile is unset", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+      },
+    });
+    // mainAgent's CALL_SITE_DEFAULTS profile is `balanced`.
+    expect(resolveDefaultProfileKey("mainAgent", llm)).toBe("balanced");
+  });
+
+  test("mainAgent falls back to catalog default when activeProfile points to a missing profile", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+      },
+    });
+    // `LLMSchema.superRefine` rejects unknown `activeProfile` references at
+    // config-load time, so this branch is unreachable for a parsed config.
+    // Mutate after parse to exercise the resolver's defensive fall-through.
+    const mutated = { ...llm, activeProfile: "does-not-exist" };
+    expect(resolveDefaultProfileKey("mainAgent", mutated)).toBe("balanced");
+  });
+
+  test("mainAgent falls back to catalog default when activeProfile is disabled", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+        gemini: {
+          provider: "gemini",
+          model: "gemini-2.5-pro",
+          status: "disabled",
+        },
+      },
+      activeProfile: "gemini",
+    });
+    expect(resolveDefaultProfileKey("mainAgent", llm)).toBe("balanced");
+  });
+
+  test("non-mainAgent ignores activeProfile and returns catalog default", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+        "cost-optimized": { provider: "openai", model: "gpt-5-mini" },
+        gemini: { provider: "gemini", model: "gemini-2.5-pro" },
+      },
+      activeProfile: "gemini",
+    });
+    // filingAgent's CALL_SITE_DEFAULTS profile is `cost-optimized` — not gemini.
+    expect(resolveDefaultProfileKey("filingAgent", llm)).toBe("cost-optimized");
+  });
+
+  test("non-mainAgent falls back to custom-* when catalog profile is missing", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        "custom-cost-optimized": {
+          provider: "openai",
+          model: "gpt-5-mini",
+        },
+      },
+    });
+    expect(resolveDefaultProfileKey("filingAgent", llm)).toBe(
+      "custom-cost-optimized",
+    );
+  });
+});
+
+describe("memory v3 call sites resolve through the standard resolver", () => {
+  const llm = LLMSchema.parse({
+    default: fullDefault,
+    profiles: {
+      balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+      "cost-optimized": {
+        provider: "anthropic",
+        model: "claude-haiku-4-5-20251001",
+      },
+    },
+  });
+
+  test("memoryV3Filter and memoryV3Descent resolve to the cost-optimized profile", () => {
+    expect(resolveDefaultProfileKey("memoryV3Filter", llm)).toBe(
+      "cost-optimized",
+    );
+    expect(resolveDefaultProfileKey("memoryV3Descent", llm)).toBe(
+      "cost-optimized",
+    );
+    expect(resolveCallSiteConfig("memoryV3Filter", llm).model).toBe(
+      "claude-haiku-4-5-20251001",
+    );
+    expect(resolveCallSiteConfig("memoryV3Descent", llm).model).toBe(
+      "claude-haiku-4-5-20251001",
+    );
+  });
+
+  test("memoryV3Gate resolves to the balanced (capable) profile", () => {
+    expect(resolveDefaultProfileKey("memoryV3Gate", llm)).toBe("balanced");
+    expect(resolveCallSiteConfig("memoryV3Gate", llm).model).toBe(
+      "claude-sonnet-4-7",
+    );
+  });
+
+  test("v3 call sites are addressable as call-site override keys", () => {
+    const overridden = LLMSchema.parse({
+      default: fullDefault,
+      callSites: {
+        memoryV3Gate: { model: "claude-opus-4-7" },
+      },
+    });
+    expect(resolveCallSiteConfig("memoryV3Gate", overridden).model).toBe(
+      "claude-opus-4-7",
+    );
+  });
+});

package/src/__tests__/llm-usage-store.test.ts

@@ -31,6 +31,7 @@ function makeInput(overrides?: Partial<UsageEventInput>): UsageEventInput {
     outputTokens: 500,
     cacheCreationInputTokens: null,
     cacheReadInputTokens: null,
+    rawUsage: null,
     actor: "main_agent",
     conversationId: null,
     runId: null,
@@ -217,6 +218,44 @@ describe("recordUsageEvent", () => {
     expect(events[0].cacheCreationInputTokens).toBe(200);
     expect(events[0].cacheReadInputTokens).toBe(300);
   });
+
+  test("round-trips the provider's raw_usage payload verbatim", () => {
+    // The `raw_usage` column carries the literal usage object the provider
+    // returned (Anthropic nests TTL breakdown under `cache_creation`,
+    // OpenAI nests cached-read details under `prompt_tokens_details`, etc.)
+    // It must round-trip from `recordUsageEvent` through SQLite back to
+    // `listUsageEvents` byte-for-byte so downstream consumers can extract
+    // any provider-specific detail without a schema change.
+    const rawUsage = {
+      input_tokens: 1000,
+      output_tokens: 500,
+      cache_creation_input_tokens: 500,
+      cache_creation: {
+        ephemeral_5m_input_tokens: 100,
+        ephemeral_1h_input_tokens: 400,
+      },
+      cache_read_input_tokens: 0,
+      service_tier: "standard",
+    };
+    recordUsageEvent(makeInput({ rawUsage }), pricedResult);
+
+    const [event] = listUsageEvents();
+    expect(event.rawUsage).toEqual(rawUsage);
+  });
+
+  test("raw_usage defaults to null when the provider did not return one", () => {
+    // Providers that don't surface a usage block (or daemons predating
+    // migration 260) write null. Coercing to `{}` would be
+    // indistinguishable from a usage block that genuinely has no fields,
+    // so we preserve null as the "absent" signal.
+    recordUsageEvent(
+      makeInput({ provider: "openai", model: "gpt-4o", rawUsage: null }),
+      pricedResult,
+    );
+
+    const [event] = listUsageEvents();
+    expect(event.rawUsage).toBeNull();
+  });
 });
 
 describe("listUsageEvents", () => {
@@ -1391,4 +1430,31 @@ describe("queryUnreportedUsageEvents", () => {
     // Analytics can treat 0 as "pre-first-turn" if needed.
     expect(events[0].turnIndex).toBe(0);
   });
+
+  test("surfaces raw_usage on unreported events", () => {
+    // The telemetry reporter consumes `queryUnreportedUsageEvents` output
+    // directly and forwards it to BigQuery. If the SELECT projection
+    // drops the `raw_usage` column, every provider-specific detail
+    // (Anthropic TTL breakdown, OpenAI prompt-token details, etc.)
+    // vanishes silently — covered here so a refactor of the projection
+    // can't regress the wire shape.
+    const rawUsage = {
+      input_tokens: 1500,
+      output_tokens: 600,
+      cache_creation_input_tokens: 800,
+      cache_creation: {
+        ephemeral_5m_input_tokens: 300,
+        ephemeral_1h_input_tokens: 500,
+      },
+    };
+    insertEventAt(1000, {
+      cacheCreationInputTokens: 800,
+      rawUsage,
+    });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].cacheCreationInputTokens).toBe(800);
+    expect(events[0].rawUsage).toEqual(rawUsage);
+  });
 });