@vellumai/assistant 0.8.2 → 0.8.4

package/src/__tests__/llm-callsite-catalog.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, test } from "bun:test";
 
+import { CALL_SITE_DEFAULTS } from "../config/call-site-defaults.js";
 import { getLLMCallSiteLabel } from "../config/llm-callsite-catalog.js";
 import { CALL_SITE_CATALOG } from "../config/schemas/call-site-catalog.js";
 import { LLMCallSiteEnum, LLMSchema } from "../config/schemas/llm.js";
@@ -50,4 +51,28 @@ describe("LLM call-site catalog", () => {
     });
     expect(parsed.callSites.memoryRouter?.model).toBe("claude-sonnet-4-6");
   });
+
+  test("CALL_SITE_DEFAULTS covers every LLMCallSite enum value", () => {
+    const defaultIds = new Set(Object.keys(CALL_SITE_DEFAULTS));
+    const missing = LLMCallSiteEnum.options.filter(
+      (id) => !defaultIds.has(id),
+    );
+    expect(missing).toEqual([]);
+  });
+
+  test("CALL_SITE_DEFAULTS contains no unknown call-site keys", () => {
+    const enumIds = new Set<string>(LLMCallSiteEnum.options);
+    const extra = Object.keys(CALL_SITE_DEFAULTS).filter(
+      (id) => !enumIds.has(id),
+    );
+    expect(extra).toEqual([]);
+  });
+
+  test("every CALL_SITE_DEFAULTS entry has a profile field", () => {
+    for (const [, config] of Object.entries(CALL_SITE_DEFAULTS)) {
+      expect(config.profile).toBeDefined();
+      expect(typeof config.profile).toBe("string");
+      expect(config.profile!.length).toBeGreaterThan(0);
+    }
+  });
 });

package/src/__tests__/llm-catalog-parity.test.ts

@@ -208,6 +208,9 @@ describe("LLM catalog parity: daemon vs client", () => {
 
   test("every provider's defaultModel exists in its models list", () => {
     for (const entry of PROVIDER_CATALOG) {
+      // Providers with an empty models list (e.g. openai-compatible) use
+      // per-connection model identifiers instead of a static catalog.
+      if (entry.models.length === 0) continue;
       const found = entry.models.some((m) => m.id === entry.defaultModel);
       expect(
         found,

package/src/__tests__/llm-context-normalization.test.ts

@@ -1152,7 +1152,6 @@ describe("normalizeLlmContextPayloads", () => {
         reasoning: { effort: "high" },
         max_output_tokens: 64000,
         stream: true,
-        store: false,
       },
       responsePayload: {
         id: "resp_abc123",
@@ -1245,7 +1244,6 @@ describe("normalizeLlmContextPayloads", () => {
           reasoning: { effort: "high" },
           max_output_tokens: 64000,
           stream: true,
-          store: false,
         },
         language: "json",
       },

package/src/__tests__/llm-request-log-agent-loop-exit-reason.test.ts

@@ -0,0 +1,116 @@
+/**
+ * Tests for `setAgentLoopExitReasonOnLatestLog` and the
+ * `agentLoopExitReason` field on the LogRow type. The helper stamps the
+ * reason onto the most-recent `llm_request_logs` row for a conversation,
+ * which is how downstream tooling distinguishes "loop kept going" (null)
+ * from "loop exited because X" (specific reason) on a per-row basis.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    ui: {},
+    model: "test",
+    provider: "test",
+    memory: { enabled: false },
+    rateLimit: { maxRequestsPerMinute: 0 },
+    secretDetection: { enabled: false },
+  }),
+}));
+
+import { getDb } from "../memory/db-connection.js";
+import { initializeDb } from "../memory/db-init.js";
+import {
+  getRequestLogById,
+  recordRequestLog,
+  setAgentLoopExitReasonOnLatestLog,
+} from "../memory/llm-request-log-store.js";
+import { llmRequestLogs } from "../memory/schema.js";
+
+initializeDb();
+
+function resetLogs(): void {
+  const db = getDb();
+  db.delete(llmRequestLogs).run();
+}
+
+describe("setAgentLoopExitReasonOnLatestLog", () => {
+  beforeEach(resetLogs);
+
+  test("recordRequestLog leaves agentLoopExitReason NULL", () => {
+    const id = recordRequestLog("conv-1", '{"req":1}', '{"res":1}');
+    const row = getRequestLogById(id);
+    expect(row).not.toBeNull();
+    expect(row!.agentLoopExitReason).toBeNull();
+  });
+
+  test("stamps the reason onto the most-recent log for the conversation", () => {
+    const first = recordRequestLog("conv-1", '{"req":1}', '{"res":1}');
+    // Ensure createdAt strict ordering — `recordRequestLog` uses
+    // `Date.now()` and bun-sqlite is fast enough that two consecutive
+    // inserts can share a millisecond. Sleep a tick to disambiguate.
+    Bun.sleepSync(2);
+    const second = recordRequestLog("conv-1", '{"req":2}', '{"res":2}');
+
+    setAgentLoopExitReasonOnLatestLog("conv-1", "no_tool_calls");
+
+    expect(getRequestLogById(first)?.agentLoopExitReason).toBeNull();
+    expect(getRequestLogById(second)?.agentLoopExitReason).toBe(
+      "no_tool_calls",
+    );
+  });
+
+  test("scopes the stamp to the given conversation only", () => {
+    const a = recordRequestLog("conv-a", '{"req":1}', '{"res":1}');
+    Bun.sleepSync(2);
+    const b = recordRequestLog("conv-b", '{"req":1}', '{"res":1}');
+
+    setAgentLoopExitReasonOnLatestLog("conv-a", "yield_to_user");
+
+    expect(getRequestLogById(a)?.agentLoopExitReason).toBe("yield_to_user");
+    // conv-b is later overall but belongs to a different conversation —
+    // must stay NULL.
+    expect(getRequestLogById(b)?.agentLoopExitReason).toBeNull();
+  });
+
+  test("no-op when conversation has no logs", () => {
+    expect(() =>
+      setAgentLoopExitReasonOnLatestLog("conv-missing", "error"),
+    ).not.toThrow();
+  });
+
+  test("does not clobber a previous run's reason when the current run never landed a row", () => {
+    // Previous run: completes, lands a log, gets stamped.
+    const prev = recordRequestLog("conv-1", '{"prev_req":1}', '{"prev_res":1}');
+    setAgentLoopExitReasonOnLatestLog("conv-1", "no_tool_calls");
+    expect(getRequestLogById(prev)?.agentLoopExitReason).toBe("no_tool_calls");
+
+    // Current run aborts pre-call (or similar) before any LLM call lands.
+    // The helper must NOT overwrite the previous run's row.
+    setAgentLoopExitReasonOnLatestLog("conv-1", "aborted_pre_call");
+    expect(getRequestLogById(prev)?.agentLoopExitReason).toBe("no_tool_calls");
+  });
+
+  test("stamps the current run's newest row even when a prior row is already stamped", () => {
+    // Prior run already stamped.
+    const prev = recordRequestLog("conv-1", '{"prev_req":1}', '{"prev_res":1}');
+    setAgentLoopExitReasonOnLatestLog("conv-1", "no_tool_calls");
+
+    // Current run lands a new log, then exits.
+    Bun.sleepSync(2);
+    const current = recordRequestLog("conv-1", '{"cur_req":1}', '{"cur_res":1}');
+    setAgentLoopExitReasonOnLatestLog("conv-1", "yield_to_user");
+
+    expect(getRequestLogById(prev)?.agentLoopExitReason).toBe("no_tool_calls");
+    expect(getRequestLogById(current)?.agentLoopExitReason).toBe(
+      "yield_to_user",
+    );
+  });
+});

package/src/__tests__/llm-request-log-error-payload.test.ts

@@ -0,0 +1,138 @@
+/**
+ * Unit tests for `buildProviderErrorResponsePayload` — the shared serializer
+ * used by `handleProviderError` (daemon) and the wake-path `onEvent` to
+ * record provider-rejected LLM calls in `llm_request_logs`.
+ *
+ * The serializer's job: take an arbitrary thrown `Error`, return a
+ * structured `{ error: {...} }` object whose fields are queryable in the
+ * LLM inspector and that round-trips cleanly through `JSON.stringify`.
+ * The `error` key wrap is load-bearing — it mirrors a successful row's
+ * `usage.rawResponse` shape so an inspector consumer can branch on
+ * `responsePayload.error` vs the success shape without re-parsing.
+ *
+ * Coverage:
+ *  - `ProviderError` with full metadata (provider, statusCode, retryAfterMs).
+ *  - `ProviderError` without optional metadata.
+ *  - Non-provider `AssistantError` (carries `code` but not provider fields).
+ *  - Plain `Error` (degrades to `{name, message}`).
+ *  - Custom `Error` subclass with overridden `name` is preserved.
+ *
+ * Each test stringifies and re-parses the payload so the on-disk shape
+ * (what eventually lands in the `responsePayload` column) is what we
+ * assert on, not the JS object identity.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import { buildProviderErrorResponsePayload } from "../memory/llm-request-log-store.js";
+import {
+  AssistantError,
+  ErrorCode,
+  ProviderError,
+} from "../util/errors.js";
+
+function persisted(err: Error): { error: Record<string, unknown> } {
+  // Round-trip through JSON to assert on the actual stored shape, not the
+  // in-memory object reference.
+  return JSON.parse(
+    JSON.stringify(buildProviderErrorResponsePayload(err)),
+  );
+}
+
+describe("buildProviderErrorResponsePayload", () => {
+  test("ProviderError with statusCode + retryAfterMs serializes every queryable field", () => {
+    const err = new ProviderError(
+      "Anthropic API error (429): rate limited",
+      "anthropic",
+      429,
+      { retryAfterMs: 1500 },
+    );
+    const got = persisted(err);
+    expect(got).toEqual({
+      error: {
+        name: "ProviderError",
+        message: "Anthropic API error (429): rate limited",
+        code: ErrorCode.PROVIDER_ERROR,
+        provider: "anthropic",
+        statusCode: 429,
+        retryAfterMs: 1500,
+      },
+    });
+  });
+
+  test("ProviderError without optional metadata omits statusCode + retryAfterMs", () => {
+    const err = new ProviderError(
+      "Gemini API error: surprise internal state",
+      "gemini",
+    );
+    const got = persisted(err);
+    expect(got).toEqual({
+      error: {
+        name: "ProviderError",
+        message: "Gemini API error: surprise internal state",
+        code: ErrorCode.PROVIDER_ERROR,
+        provider: "gemini",
+      },
+    });
+    // Explicit assertion: omitted fields aren't present as `null` either —
+    // the inspector should be able to test `'statusCode' in error` reliably.
+    expect("statusCode" in got.error).toBe(false);
+    expect("retryAfterMs" in got.error).toBe(false);
+  });
+
+  test("non-provider AssistantError carries the ErrorCode but no provider fields", () => {
+    // Tool errors / permission denials are technically also AssistantErrors;
+    // we just want to confirm the generic AssistantError branch produces a
+    // sensible row rather than silently degrading to a plain Error shape.
+    const err = new AssistantError(
+      "internal state corrupted",
+      ErrorCode.INTERNAL_ERROR,
+    );
+    const got = persisted(err);
+    expect(got).toEqual({
+      error: {
+        name: "AssistantError",
+        message: "internal state corrupted",
+        code: ErrorCode.INTERNAL_ERROR,
+      },
+    });
+    expect("provider" in got.error).toBe(false);
+  });
+
+  test("plain Error degrades to {name, message} with no code/provider noise", () => {
+    const err = new Error("connection reset");
+    const got = persisted(err);
+    expect(got).toEqual({
+      error: {
+        name: "Error",
+        message: "connection reset",
+      },
+    });
+    expect("code" in got.error).toBe(false);
+  });
+
+  test("custom Error subclass with overridden name is preserved", () => {
+    class TimeoutError extends Error {
+      constructor(message: string) {
+        super(message);
+        this.name = "TimeoutError";
+      }
+    }
+    const got = persisted(new TimeoutError("provider timed out after 60s"));
+    expect(got).toEqual({
+      error: {
+        name: "TimeoutError",
+        message: "provider timed out after 60s",
+      },
+    });
+  });
+
+  test("ProviderError with statusCode 0 is still recorded (not coerced to undefined)", () => {
+    // Defensive: `if (err.statusCode !== undefined)` correctly admits 0.
+    // A raw `if (err.statusCode)` would drop it, so the test guards against
+    // a regression to truthy-checking.
+    const err = new ProviderError("weird provider", "fake", 0);
+    const got = persisted(err);
+    expect(got.error.statusCode).toBe(0);
+  });
+});

package/src/__tests__/llm-request-log-source-clickhouse.test.ts

@@ -83,6 +83,7 @@ const SAMPLE_ROW = {
   response_payload: '{"bar":2}',
   // ClickHouse emits Int64 as a quoted string under JSONEachRow by default.
   created_at: "1778465138786",
+  agent_loop_exit_reason: "no_tool_calls",
 };
 
 describe("ClickHouseLlmRequestLogSource", () => {
@@ -102,6 +103,7 @@ describe("ClickHouseLlmRequestLogSource", () => {
       requestPayload: '{"foo":1}',
       responsePayload: '{"bar":2}',
       createdAt: 1778465138786,
+      agentLoopExitReason: "no_tool_calls",
     });
   });