@vellumai/assistant 0.7.3 → 0.8.1

package/src/__tests__/llm-request-log-source-clickhouse.test.ts

@@ -0,0 +1,188 @@
+import { describe, expect, mock, test } from "bun:test";
+
+function makeLoggerStub(): Record<string, unknown> {
+  const stub: Record<string, unknown> = {};
+  for (const m of [
+    "info",
+    "warn",
+    "error",
+    "debug",
+    "trace",
+    "fatal",
+    "silent",
+    "child",
+  ]) {
+    stub[m] = m === "child" ? () => makeLoggerStub() : () => {};
+  }
+  return stub;
+}
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeLoggerStub(),
+}));
+
+import { ClickHouseLlmRequestLogSource } from "../memory/llm-request-log-source-clickhouse.js";
+
+const DEFAULT_CONFIG = {
+  database: "default",
+  table: "llm_request_logs",
+  user: "default",
+};
+
+interface FakeFetchCall {
+  url: string;
+  init: RequestInit | undefined;
+}
+
+function fakeFetchReturning(
+  body: string,
+  status = 200,
+  recorder?: FakeFetchCall[],
+): typeof fetch {
+  return ((url: string | URL | Request, init?: RequestInit) => {
+    recorder?.push({
+      url: typeof url === "string" ? url : url.toString(),
+      init,
+    });
+    return Promise.resolve(
+      new Response(body, {
+        status,
+        headers: { "content-type": "application/x-ndjson" },
+      }),
+    );
+  }) as typeof fetch;
+}
+
+function makeSource(opts: {
+  body?: string;
+  status?: number;
+  recorder?: FakeFetchCall[];
+  resolveTurnMessageIds?: (messageId: string) => string[];
+  resolveMessage?: (messageId: string) => { metadata: string | null } | null;
+}) {
+  return new ClickHouseLlmRequestLogSource(DEFAULT_CONFIG, {
+    resolveUrl: async () => "https://ch.example.test:8443",
+    resolvePassword: async () => "hunter2",
+    resolveAssistantId: async () => "asst-fixture-001",
+    resolveTurnMessageIds: opts.resolveTurnMessageIds ?? (() => []),
+    resolveMessage: opts.resolveMessage ?? (() => null),
+    fetchImpl: fakeFetchReturning(
+      opts.body ?? "",
+      opts.status ?? 200,
+      opts.recorder,
+    ),
+  });
+}
+
+const SAMPLE_ROW = {
+  id: "log-1",
+  conversation_id: "conv-1",
+  message_id: "msg-1",
+  provider: "anthropic",
+  request_payload: '{"foo":1}',
+  response_payload: '{"bar":2}',
+  // ClickHouse emits Int64 as a quoted string under JSONEachRow by default.
+  created_at: "1778465138786",
+};
+
+describe("ClickHouseLlmRequestLogSource", () => {
+  test("getRequestLogById returns null on empty response", async () => {
+    const src = makeSource({ body: "" });
+    expect(await src.getRequestLogById("missing")).toBeNull();
+  });
+
+  test("getRequestLogById parses a single JSONEachRow row into LogRow", async () => {
+    const src = makeSource({ body: JSON.stringify(SAMPLE_ROW) + "\n" });
+    const row = await src.getRequestLogById("log-1");
+    expect(row).toEqual({
+      id: "log-1",
+      conversationId: "conv-1",
+      messageId: "msg-1",
+      provider: "anthropic",
+      requestPayload: '{"foo":1}',
+      responsePayload: '{"bar":2}',
+      createdAt: 1778465138786,
+    });
+  });
+
+  test("maps empty-string message_id and provider back to null", async () => {
+    const src = makeSource({
+      body:
+        JSON.stringify({
+          ...SAMPLE_ROW,
+          message_id: "",
+          provider: "",
+        }) + "\n",
+    });
+    const row = await src.getRequestLogById("log-1");
+    expect(row?.messageId).toBeNull();
+    expect(row?.provider).toBeNull();
+  });
+
+  test("sets the database URL param + assistant_id query param + Basic auth", async () => {
+    const recorder: FakeFetchCall[] = [];
+    const src = makeSource({
+      body: JSON.stringify(SAMPLE_ROW) + "\n",
+      recorder,
+    });
+    await src.getRequestLogById("log-1");
+    expect(recorder).toHaveLength(1);
+    const call = recorder[0]!;
+    const parsed = new URL(call.url);
+    expect(parsed.searchParams.get("database")).toBe("default");
+    expect(parsed.searchParams.get("param_assistant_id")).toBe("asst-fixture-001");
+    expect(parsed.searchParams.get("param_log_id")).toBe("log-1");
+    expect(call.init?.method).toBe("POST");
+    const auth = (call.init?.headers as Record<string, string>).Authorization;
+    expect(auth).toMatch(/^Basic /);
+    const decoded = Buffer.from(auth.slice("Basic ".length), "base64").toString(
+      "utf8",
+    );
+    expect(decoded).toBe("default:hunter2");
+    // The SQL body should reference the configured table and the parameterized
+    // placeholders ClickHouse will substitute from `param_*`.
+    expect(call.init?.body).toContain("`llm_request_logs`");
+    expect(call.init?.body).toContain("{assistant_id:String}");
+    expect(call.init?.body).toContain("{log_id:String}");
+  });
+
+  test("throws a descriptive error on non-2xx", async () => {
+    const src = makeSource({ body: "boom", status: 503 });
+    await expect(src.getRequestLogById("log-1")).rejects.toThrow(
+      /ClickHouse query failed \(HTTP 503\): boom/,
+    );
+  });
+
+  test("getRequestLogsByMessageId returns [] when turn resolution yields no message IDs", async () => {
+    // The module-level mock of conversation-crud returns [] for the turn,
+    // and null for fork-source resolution, so this exercises the empty path
+    // without any database setup.
+    const src = makeSource({ body: "" });
+    const rows = await src.getRequestLogsByMessageId("msg-x");
+    expect(rows).toEqual([]);
+  });
+
+  test("missing clickhouse:url credential surfaces a clear error", async () => {
+    const src = new ClickHouseLlmRequestLogSource(DEFAULT_CONFIG, {
+      resolveUrl: async () => null,
+      resolvePassword: async () => "x",
+      resolveAssistantId: async () => "asst-1",
+      fetchImpl: fakeFetchReturning(""),
+    });
+    await expect(src.getRequestLogById("log-1")).rejects.toThrow(
+      /clickhouse:url credential is required/,
+    );
+  });
+
+  test("missing vellum:platform_assistant_id surfaces a clear error", async () => {
+    const src = new ClickHouseLlmRequestLogSource(DEFAULT_CONFIG, {
+      resolveUrl: async () => "https://ch.example.test",
+      resolvePassword: async () => "x",
+      resolveAssistantId: async () => null,
+      fetchImpl: fakeFetchReturning(""),
+    });
+    await expect(src.getRequestLogById("log-1")).rejects.toThrow(
+      /vellum:platform_assistant_id credential is required/,
+    );
+  });
+});

package/src/__tests__/llm-request-log-source-factory.test.ts

@@ -0,0 +1,124 @@
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+function makeLoggerStub(): Record<string, unknown> {
+  const stub: Record<string, unknown> = {};
+  for (const m of [
+    "info",
+    "warn",
+    "error",
+    "debug",
+    "trace",
+    "fatal",
+    "silent",
+    "child",
+  ]) {
+    stub[m] = m === "child" ? () => makeLoggerStub() : () => {};
+  }
+  return stub;
+}
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeLoggerStub(),
+}));
+
+// Configurable config stub. Each test sets `currentConfig` before calling
+// the factory. The ClickHouse source is dynamic-imported, so its constructor
+// runs lazily; we don't need to mock conversation-crud here because none of
+// the factory's tests actually exercise the CH read path.
+const LOCAL_DEFAULT_CONFIG = {
+  llmRequestLogs: {
+    readSource: "local",
+    clickhouse: {
+      database: "default",
+      table: "llm_request_logs",
+      user: "default",
+    },
+  },
+};
+let currentConfig: unknown = LOCAL_DEFAULT_CONFIG;
+mock.module("../config/loader.js", () => ({
+  getConfig: () => currentConfig,
+}));
+
+// Bun's `mock.module()` persists process-wide; reset to the safe local
+// default after this file runs so other test files that touch this module
+// (or transitively call `getConfig()` through the factory) don't pick up
+// a stale `readSource=clickhouse` value. Also drop the factory's internal
+// cache so the next caller resolves fresh against the reset config.
+afterAll(() => {
+  currentConfig = LOCAL_DEFAULT_CONFIG;
+  invalidateLlmRequestLogSourceCache();
+});
+
+import {
+  getLlmRequestLogSource,
+  invalidateLlmRequestLogSourceCache,
+} from "../memory/llm-request-log-source.js";
+import { ClickHouseLlmRequestLogSource } from "../memory/llm-request-log-source-clickhouse.js";
+import { LocalLlmRequestLogSource } from "../memory/llm-request-log-source-local.js";
+
+describe("getLlmRequestLogSource factory", () => {
+  beforeEach(() => {
+    invalidateLlmRequestLogSourceCache();
+  });
+
+  test("returns LocalLlmRequestLogSource when readSource is 'local'", async () => {
+    currentConfig = {
+      llmRequestLogs: {
+        readSource: "local",
+        clickhouse: { database: "default", table: "llm_request_logs", user: "default" },
+      },
+    };
+    const src = await getLlmRequestLogSource();
+    expect(src).toBeInstanceOf(LocalLlmRequestLogSource);
+  });
+
+  test("returns LocalLlmRequestLogSource when llmRequestLogs is undefined (defensive)", async () => {
+    currentConfig = {};
+    const src = await getLlmRequestLogSource();
+    expect(src).toBeInstanceOf(LocalLlmRequestLogSource);
+  });
+
+  test("returns ClickHouseLlmRequestLogSource when readSource is 'clickhouse'", async () => {
+    currentConfig = {
+      llmRequestLogs: {
+        readSource: "clickhouse",
+        clickhouse: { database: "default", table: "llm_request_logs", user: "default" },
+      },
+    };
+    const src = await getLlmRequestLogSource();
+    expect(src).toBeInstanceOf(ClickHouseLlmRequestLogSource);
+  });
+
+  test("caches the resolved source across calls", async () => {
+    currentConfig = {
+      llmRequestLogs: {
+        readSource: "local",
+        clickhouse: { database: "default", table: "llm_request_logs", user: "default" },
+      },
+    };
+    const first = await getLlmRequestLogSource();
+    const second = await getLlmRequestLogSource();
+    expect(second).toBe(first);
+  });
+
+  test("invalidateLlmRequestLogSourceCache forces a fresh resolution", async () => {
+    currentConfig = {
+      llmRequestLogs: {
+        readSource: "local",
+        clickhouse: { database: "default", table: "llm_request_logs", user: "default" },
+      },
+    };
+    const first = await getLlmRequestLogSource();
+    invalidateLlmRequestLogSourceCache();
+    currentConfig = {
+      llmRequestLogs: {
+        readSource: "clickhouse",
+        clickhouse: { database: "default", table: "llm_request_logs", user: "default" },
+      },
+    };
+    const second = await getLlmRequestLogSource();
+    expect(second).not.toBe(first);
+    expect(second).toBeInstanceOf(ClickHouseLlmRequestLogSource);
+  });
+});

package/src/__tests__/llm-resolver.test.ts

@@ -52,6 +52,50 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.maxTokens).toBe(64000);
   });
 
+  test("model-only call-site override infers provider from known model owner", () => {
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+      },
+      profiles: {
+        active: { provider: "openai", model: "gpt-5.5" },
+      },
+      activeProfile: "active",
+      callSites: {
+        conversationStarters: {
+          model: "claude-haiku-4-5-20251001",
+          effort: "low",
+        },
+      },
+    });
+
+    const resolved = resolveCallSiteConfig("conversationStarters", llm);
+
+    expect(resolved.provider).toBe("anthropic");
+    expect(resolved.model).toBe("claude-haiku-4-5-20251001");
+    expect(resolved.effort).toBe("low");
+  });
+
+  test("unknown model-only override preserves inherited provider", () => {
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+      },
+      callSites: {
+        memoryExtraction: { model: "local-custom-model" },
+      },
+    });
+
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+
+    expect(resolved.provider).toBe("openai");
+    expect(resolved.model).toBe("local-custom-model");
+  });
+
   test("profile field overrides default when call site references it", () => {
     const llm = LLMSchema.parse({
       default: fullDefault,
@@ -207,6 +251,7 @@ describe("resolveCallSiteConfig", () => {
       callSites: {
         mainAgent: { profile: "nonexistent" },
       },
+      profileSession: { defaultTtlSeconds: 1800, maxTtlSeconds: 43200 },
       pricingOverrides: [],
     };
     expect(() => resolveCallSiteConfig("mainAgent", llm)).toThrow(
@@ -339,6 +384,7 @@ describe("resolveCallSiteConfig", () => {
       profileOrder: [],
       callSites: {},
       activeProfile: "nonexistent",
+      profileSession: { defaultTtlSeconds: 1800, maxTtlSeconds: 43200 },
       pricingOverrides: [],
     };
     const resolved = resolveCallSiteConfig("mainAgent", llm);

package/src/__tests__/managed-profile-guard.test.ts

@@ -96,14 +96,15 @@ beforeEach(() => {
 // ---------------------------------------------------------------------------
 
 describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
-  test("rejects edits to quality-optimized with descriptive message", () => {
+  test("rejects edits to quality-optimized that touch non-label/status fields", () => {
     expect(() =>
       replaceRoute.handler({
         pathParams: { name: "quality-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
     ).toThrow(
-      'Cannot edit managed profile "quality-optimized". Duplicate it to create a custom profile.',
+      'Cannot edit managed profile "quality-optimized" fields [provider, model]. ' +
+        "Only label and status may be edited; duplicate to a custom profile to change other fields.",
     );
   });
 
@@ -144,6 +145,134 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(result).toEqual({ ok: true });
     expect(savedRaw).not.toBeNull();
   });
+
+  // -------------------------------------------------------------------------
+  // Null-as-clear sentinel: `patchManagedProfileFields` has handled `null`
+  // for label and status since #30362, but the Zod `ProfileEntry` schema
+  // had them as `.optional()` (not `.nullable()`), which rejected null
+  // payloads at parse time before the route handler ever saw them.
+  // These tests lock the round-trip now that the schema accepts null.
+  // -------------------------------------------------------------------------
+
+  test("PUT { label: null } on managed profile clears the label on disk", () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          balanced: {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            label: "My Custom Name",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = replaceRoute.handler({
+      pathParams: { name: "balanced" },
+      body: { label: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm?.profiles
+      ?.balanced as Record<string, unknown>;
+    // Label key removed; seed fields preserved.
+    expect("label" in profile).toBe(false);
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-sonnet");
+    expect(profile.source).toBe("managed");
+  });
+
+  test("PUT { status: null } on managed profile clears status (back to active-by-absence)", () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          "quality-optimized": {
+            provider: "anthropic",
+            model: "claude-opus",
+            status: "disabled",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = replaceRoute.handler({
+      pathParams: { name: "quality-optimized" },
+      body: { status: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm
+      ?.profiles?.["quality-optimized"] as Record<string, unknown>;
+    expect("status" in profile).toBe(false);
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-opus");
+  });
+
+  test("PUT { label: null, status: null } clears both in a single request", () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          "cost-optimized": {
+            provider: "anthropic",
+            model: "claude-haiku",
+            label: "Speed (Custom)",
+            status: "disabled",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = replaceRoute.handler({
+      pathParams: { name: "cost-optimized" },
+      body: { label: null, status: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm
+      ?.profiles?.["cost-optimized"] as Record<string, unknown>;
+    expect("label" in profile).toBe(false);
+    expect(profile.status).toBeUndefined();
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-haiku");
+  });
+
+  test("PUT { label: null, status: 'disabled' } mixes clear + set in one call", () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          balanced: {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            label: "Custom Label",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = replaceRoute.handler({
+      pathParams: { name: "balanced" },
+      body: { label: null, status: "disabled" },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm?.profiles
+      ?.balanced as Record<string, unknown>;
+    expect("label" in profile).toBe(false);
+    expect(profile.status).toBe("disabled");
+  });
+
+  test("PUT { label: '' } on managed profile still rejected by `.min(1)`", () => {
+    // `.nullable()` only widens the type to accept null — empty strings
+    // still fail the min-length check, which is correct: an empty string
+    // would persist as a literal "" override, not the clear-to-seed
+    // intent. Clients must send `null` to clear.
+    expect(() =>
+      replaceRoute.handler({
+        pathParams: { name: "balanced" },
+        body: { label: "" },
+      }),
+    ).toThrow(BadRequestError);
+  });
 });
 
 // ---------------------------------------------------------------------------

package/src/__tests__/mcp-auth-routes.test.ts

@@ -36,6 +36,7 @@ mock.module("../config/loader.js", () => ({
       },
     },
   }),
+  saveRawConfig: () => {},
 }));
 
 mock.module("../util/logger.js", () => ({