@vellumai/assistant 0.10.7-dev.202607101343.e727d4c → 0.10.7-dev.202607101554.2aa9767

package/docs/workspace-tools.md

@@ -130,14 +130,17 @@ on the incoming tool.
 initializeTools()             # core tools register
   → loadWorkspaceTools()      # initial workspace tool reconcile
     → MCP tool registration
-    → loadUserPlugins()
+    → loadUserPlugins()       # populates the plugin mtime-cache (no registry writes)
+    → loadPluginTools()       # registry pulls the active user-plugin tool set
     → bootstrapPlugins()
 
 # on every conversation turn (createResolveToolsCallback):
 resolveTools(history)
   → loadWorkspaceTools()                # reconcile registry against disk (fire-and-forget)
+  → loadPluginTools()                   # pull user-plugin tools from the mtime-cache (same pattern)
   → getWorkspaceToolDefinitions()       # re-read workspace tools from the registry
   → getMcpToolDefinitions()             # re-read MCP tools (same pattern)
+  → getPluginToolDefinitions()          # re-read plugin tools from the registry
 ```
 
 Workspace tools register _after_ core tools and _before_ every other

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.10.7-dev.202607101343.e727d4c",
+  "version": "0.10.7-dev.202607101554.2aa9767",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/conversation-agent-loop.test.ts

@@ -1606,6 +1606,31 @@ describe("session-agent-loop", () => {
       expect(mainAgentCall?.[2]).toBe(3);
       expect(mainAgentCall?.[3]).toBe("gpt-4.1-2026-03-01");
     });
+
+    test("persists the served model onto the assistant row's metadata at finalize", async () => {
+      const events: ServerMessage[] = [];
+
+      const ctx = makeCtx({
+        providerResponses: [
+          {
+            content: [{ type: "text", text: "Hi there." }],
+            model: "gpt-4.1-2026-03-01",
+            usage: { inputTokens: 12, outputTokens: 3 },
+            stopReason: "end_turn",
+          },
+        ],
+      });
+
+      await runAgentLoopImpl(ctx, "hello", "msg-1", (msg) => events.push(msg));
+
+      // The finalize write carries the `message_complete` event's model
+      // (`response.model`) as metadata alongside the content, in one write.
+      const finalizeCall = updateMessageContentMock.mock.calls.find(
+        (call) => (call as unknown[])[2] !== undefined,
+      ) as unknown[] | undefined;
+      expect(finalizeCall).toBeDefined();
+      expect(finalizeCall?.[2]).toEqual({ model: "gpt-4.1-2026-03-01" });
+    });
   });
 
   describe("checkpoint handoff (infinite loop prevention)", () => {

package/src/__tests__/credential-security-invariants.test.ts

@@ -208,6 +208,7 @@ describe("Invariant 2: no generic plaintext secret read API", () => {
       "media/image-credentials.ts", // shared image-gen credential resolver (provider API key lookup)
       "persistence/embeddings/embedding-backend.ts", // embedding backend API key lookup
       "persistence/llm-request-log-source-clickhouse.ts", // ClickHouse read source — lazy lookup of clickhouse:url + clickhouse:password + vellum:platform_assistant_id for self-scoped mirror reads
+      "persistence/llm-request-log-sink-clickhouse.ts", // ClickHouse write sink — lazy lookup of clickhouse:url + clickhouse:password + vellum:platform_assistant_id for self-scoped log writes
       "persistence/compaction-log-store-clickhouse.ts", // ClickHouse compaction log writer — lazy lookup of clickhouse:url + clickhouse:password + vellum:platform_assistant_id for self-scoped event writes
       "daemon/providers-setup.ts", // provider initialization API key lookup
       "workspace/migrations/006-services-config.ts", // services config migration reads provider API keys

package/src/__tests__/llm-request-log-clickhouse-routing.test.ts

@@ -0,0 +1,117 @@
+/**
+ * Tests that the store's write path routes to ClickHouse instead of local
+ * SQLite when a ClickHouse sink is configured — and stays on SQLite
+ * otherwise. The sink module is mocked so the routing decision is observable
+ * without a live ClickHouse.
+ */
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+// Logging stays enabled; `getConfigReadOnly` is only consulted by the store.s
+// disabled gate here (the sink factory itself is mocked below).
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ llmRequestLogs: { readSource: "local" } }),
+  getConfigReadOnly: () => ({ llmRequestLogs: { readSource: "local" } }),
+}));
+
+// Mutable fake sink: when non-null, the store must route to it.
+interface CapturedRow {
+  id: string;
+  conversationId: string;
+  callSite: string | null;
+}
+let capturedRows: CapturedRow[] = [];
+let sinkActive = false;
+mock.module("../persistence/llm-request-log-sink-clickhouse.js", () => ({
+  getClickHouseLlmRequestLogSink: () =>
+    sinkActive
+      ? {
+          recordBestEffort: (row: CapturedRow) => {
+            capturedRows.push(row);
+          },
+        }
+      : null,
+}));
+
+afterAll(() => {
+  sinkActive = false;
+});
+
+import { getLogsDb } from "../persistence/db-connection.js";
+import { initializeDb } from "../persistence/db-init.js";
+import {
+  getRequestLogsByConversationId,
+  recordRequestLog,
+  recordSyntheticAgentErrorMessageLog,
+} from "../persistence/llm-request-log-store.js";
+import { llmRequestLogs } from "../persistence/schema/index.js";
+
+await initializeDb();
+
+function resetLogs(): void {
+  getLogsDb()!.delete(llmRequestLogs).run();
+}
+
+describe("write routing between SQLite and ClickHouse", () => {
+  beforeEach(() => {
+    resetLogs();
+    capturedRows = [];
+    sinkActive = false;
+  });
+
+  test("writes to local SQLite when no ClickHouse sink is configured", () => {
+    const id = recordRequestLog(
+      "conv-local",
+      '{"req":1}',
+      '{"res":1}',
+      undefined,
+      "anthropic",
+      "mainAgent",
+    );
+    expect(id).not.toBeNull();
+    expect(capturedRows).toHaveLength(0);
+    const rows = getRequestLogsByConversationId("conv-local");
+    expect(rows).toHaveLength(1);
+    expect(rows[0]!.provider).toBe("anthropic");
+  });
+
+  test("routes to ClickHouse and skips SQLite when a sink is configured", () => {
+    sinkActive = true;
+    const id = recordRequestLog(
+      "conv-ch",
+      '{"req":1}',
+      '{"res":1}',
+      "msg-ch",
+      "openai",
+      "mainAgent",
+    );
+    expect(id).not.toBeNull();
+    // Nothing landed in the local table.
+    expect(getRequestLogsByConversationId("conv-ch")).toEqual([]);
+    // The row went to the sink, carrying the same id and call site.
+    expect(capturedRows).toHaveLength(1);
+    expect(capturedRows[0]!.id).toBe(id!);
+    expect(capturedRows[0]!.conversationId).toBe("conv-ch");
+    expect(capturedRows[0]!.callSite).toBe("mainAgent");
+  });
+
+  test("routes synthetic error rows to ClickHouse too", () => {
+    sinkActive = true;
+    const id = recordSyntheticAgentErrorMessageLog({
+      conversationId: "conv-ch-2",
+      messageId: "msg-ch-2",
+      exitReason: "budget_yield_unrecovered",
+      noticeText: "Out of budget.",
+      preparedRequest: null,
+      createdAt: 1_700_000_000_000,
+    });
+    expect(id).not.toBeNull();
+    expect(getRequestLogsByConversationId("conv-ch-2")).toEqual([]);
+    expect(capturedRows).toHaveLength(1);
+    expect(capturedRows[0]!.callSite).toBe("syntheticAgentErrorMessage");
+  });
+});

package/src/__tests__/llm-request-log-sink-clickhouse.test.ts

@@ -0,0 +1,219 @@
+/**
+ * Tests the ClickHouse WRITE sink for LLM request logs: the wire shape it
+ * POSTs (lazy CREATE TABLE + JSONEachRow INSERT, null → '' sentinels,
+ * assistant_id injected from credentials) and the config-gated factory.
+ */
+import { afterEach, describe, expect, mock, test } from "bun:test";
+
+function makeLoggerStub(): Record<string, unknown> {
+  const stub: Record<string, unknown> = {};
+  for (const m of [
+    "info",
+    "warn",
+    "error",
+    "debug",
+    "trace",
+    "fatal",
+    "silent",
+    "child",
+  ]) {
+    stub[m] = m === "child" ? () => makeLoggerStub() : () => {};
+  }
+  return stub;
+}
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeLoggerStub(),
+}));
+
+// Mutable config the factory reads via `getConfigReadOnly`.
+let currentConfig: unknown = { llmRequestLogs: { readSource: "local" } };
+mock.module("../config/loader.js", () => ({
+  getConfigReadOnly: () => currentConfig,
+}));
+
+afterEach(() => {
+  currentConfig = { llmRequestLogs: { readSource: "local" } };
+  resetClickHouseLlmRequestLogSinkForTests();
+});
+
+import {
+  ClickHouseLlmRequestLogSink,
+  getClickHouseLlmRequestLogSink,
+  resetClickHouseLlmRequestLogSinkForTests,
+} from "../persistence/llm-request-log-sink-clickhouse.js";
+
+const DEFAULT_CONFIG = {
+  database: "default",
+  table: "llm_request_logs",
+  user: "default",
+};
+
+interface FakeFetchCall {
+  url: string;
+  init: RequestInit | undefined;
+}
+
+function fakeFetch(recorder: FakeFetchCall[], status = 200): typeof fetch {
+  return ((url: string | URL | Request, init?: RequestInit) => {
+    recorder.push({
+      url: typeof url === "string" ? url : url.toString(),
+      init,
+    });
+    return Promise.resolve(new Response("", { status }));
+  }) as typeof fetch;
+}
+
+function makeSink(recorder: FakeFetchCall[], status = 200) {
+  return new ClickHouseLlmRequestLogSink(DEFAULT_CONFIG, {
+    resolveUrl: async () => "https://ch.example.test:8443",
+    resolvePassword: async () => "hunter2",
+    resolveAssistantId: async () => "asst-fixture-001",
+    fetchImpl: fakeFetch(recorder, status),
+  });
+}
+
+describe("ClickHouseLlmRequestLogSink.insert", () => {
+  test("creates the table then inserts the JSONEachRow wire shape", async () => {
+    const calls: FakeFetchCall[] = [];
+    const sink = makeSink(calls);
+
+    await sink.insert({
+      id: "log-1",
+      conversationId: "conv-1",
+      messageId: "msg-1",
+      provider: "anthropic",
+      requestPayload: '{"req":1}',
+      responsePayload: '{"res":1}',
+      createdAt: 1_700_000_000_123,
+      agentLoopExitReason: "no_tool_calls",
+      callSite: "mainAgent",
+    });
+
+    expect(calls).toHaveLength(2);
+    // 1) DDL as the request body, no `query` param.
+    expect(String(calls[0]!.init?.body)).toContain(
+      "CREATE TABLE IF NOT EXISTS",
+    );
+    expect(new URL(calls[0]!.url).searchParams.get("query")).toBeNull();
+
+    // 2) INSERT statement in the `query` param, row JSON in the body.
+    const insertUrl = new URL(calls[1]!.url);
+    expect(insertUrl.searchParams.get("query")).toContain(
+      "INSERT INTO `llm_request_logs` FORMAT JSONEachRow",
+    );
+    expect(insertUrl.searchParams.get("database")).toBe("default");
+    const row = JSON.parse(String(calls[1]!.init?.body));
+    expect(row).toEqual({
+      assistant_id: "asst-fixture-001",
+      id: "log-1",
+      conversation_id: "conv-1",
+      message_id: "msg-1",
+      provider: "anthropic",
+      request_payload: '{"req":1}',
+      response_payload: '{"res":1}',
+      created_at: 1_700_000_000_123,
+      agent_loop_exit_reason: "no_tool_calls",
+      call_site: "mainAgent",
+    });
+    // Basic auth is sent.
+    expect(
+      (calls[1]!.init?.headers as Record<string, string>).Authorization,
+    ).toStartWith("Basic ");
+  });
+
+  test("maps null string fields to the '' sentinel", async () => {
+    const calls: FakeFetchCall[] = [];
+    const sink = makeSink(calls);
+
+    await sink.insert({
+      id: "log-2",
+      conversationId: "conv-2",
+      messageId: null,
+      provider: null,
+      requestPayload: "{}",
+      responsePayload: "{}",
+      createdAt: 1,
+      agentLoopExitReason: null,
+      callSite: null,
+    });
+
+    const row = JSON.parse(String(calls[1]!.init?.body));
+    expect(row.message_id).toBe("");
+    expect(row.provider).toBe("");
+    expect(row.agent_loop_exit_reason).toBe("");
+    expect(row.call_site).toBe("");
+  });
+
+  test("insert rejects on a ClickHouse error status", async () => {
+    const calls: FakeFetchCall[] = [];
+    const sink = makeSink(calls, 500);
+    await expect(
+      sink.insert({
+        id: "log-3",
+        conversationId: "conv-3",
+        messageId: null,
+        provider: null,
+        requestPayload: "{}",
+        responsePayload: "{}",
+        createdAt: 1,
+        agentLoopExitReason: null,
+        callSite: null,
+      }),
+    ).rejects.toThrow(/ClickHouse llm-request-log write failed/);
+  });
+
+  test("recordBestEffort never throws synchronously on a failing backend", () => {
+    const calls: FakeFetchCall[] = [];
+    const sink = makeSink(calls, 500);
+    expect(() =>
+      sink.recordBestEffort({
+        id: "log-4",
+        conversationId: "conv-4",
+        messageId: null,
+        provider: null,
+        requestPayload: "{}",
+        responsePayload: "{}",
+        createdAt: 1,
+        agentLoopExitReason: null,
+        callSite: null,
+      }),
+    ).not.toThrow();
+  });
+});
+
+describe("getClickHouseLlmRequestLogSink factory", () => {
+  test("returns null when readSource is 'local'", () => {
+    currentConfig = { llmRequestLogs: { readSource: "local" } };
+    expect(getClickHouseLlmRequestLogSink()).toBeNull();
+  });
+
+  test("returns null when llmRequestLogs is absent", () => {
+    currentConfig = {};
+    expect(getClickHouseLlmRequestLogSink()).toBeNull();
+  });
+
+  test("returns a sink when readSource is 'clickhouse'", () => {
+    currentConfig = {
+      llmRequestLogs: { readSource: "clickhouse", clickhouse: DEFAULT_CONFIG },
+    };
+    expect(getClickHouseLlmRequestLogSink()).toBeInstanceOf(
+      ClickHouseLlmRequestLogSink,
+    );
+  });
+
+  test("re-instantiates when the clickhouse config changes", () => {
+    currentConfig = {
+      llmRequestLogs: { readSource: "clickhouse", clickhouse: DEFAULT_CONFIG },
+    };
+    const first = getClickHouseLlmRequestLogSink();
+    currentConfig = {
+      llmRequestLogs: {
+        readSource: "clickhouse",
+        clickhouse: { ...DEFAULT_CONFIG, table: "other_logs" },
+      },
+    };
+    const second = getClickHouseLlmRequestLogSink();
+    expect(second).not.toBe(first);
+  });
+});

package/src/__tests__/mtime-cache.test.ts

@@ -43,9 +43,11 @@ import {
 } from "../plugins/mtime-cache.js";
 import { getSourceVersionsPath } from "../plugins/source-versions.js";
 import {
+  __resetRegistryForTesting,
   getAllToolDefinitions,
   getPluginToolDefinitions,
   getToolOwner,
+  loadPluginTools,
 } from "../tools/registry.js";
 
 // ─── Test fixtures ───────────────────────────────────────────────────────────
@@ -171,6 +173,9 @@ beforeEach(() => {
   watchState = null;
   sentinelTouchSeq = 0;
   resetPluginCacheForTests();
+  // The registry pulls plugin tools via loadPluginTools(); clear its side
+  // (tools + pulled fingerprints) so registrations never leak across tests.
+  __resetRegistryForTesting();
 });
 
 afterAll(() => {
@@ -357,9 +362,11 @@ describe("plugin mtime cache (per-surface)", () => {
     touchFile(toolFile);
     publishSourceChanges();
     await getUserHooksFor("user-prompt-submit");
+    await loadPluginTools();
 
     // Both the cache and the global registry serve the fresh definition —
-    // the redeploy unregistered the old tool and registered the new one.
+    // the moved source mtime changed the plugin's fingerprint, so the pull
+    // reconcile re-registered its tool set.
     expect(getCachedUserTools()[0]?.description).toBe("v2");
     expect(
       getAllToolDefinitions().find((t) => t.name === "edited-tool")
@@ -632,11 +639,13 @@ const TOOL_SRC = (name: string) =>
  * production sequence: the monitor's pass rewrites the sentinel, and the
  * next hook dispatch's one-stat gate applies the diff. The hook name is
  * irrelevant — the reconcile runs regardless of whether a plugin defines
- * that hook.
+ * that hook. Finish with the registry pull the per-turn tool resolver
+ * fires, so registry-facing assertions see the reconciled tool set.
  */
 async function publishAndDispatch(): Promise<void> {
   publishSourceChanges();
   await getUserHooksFor("user-prompt-submit");
+  await loadPluginTools();
 }
 
 describe("plugin runtime activation", () => {
@@ -701,6 +710,7 @@ describe("plugin runtime activation", () => {
     writeMarkerHook(dir, "shutdown", shutdownMarker, "bye");
 
     await populateCacheAtBoot();
+    await loadPluginTools(); // boot pull (initializePlugins does this)
     expect(getToolOwner("temp-tool")?.kind).toBe("plugin");
 
     rmSync(dir, { recursive: true, force: true });
@@ -740,6 +750,7 @@ describe("plugin runtime activation", () => {
     writeMarkerHook(dir, "shutdown", shutdownMarker, "disabled");
 
     await populateCacheAtBoot();
+    await loadPluginTools(); // boot pull (initializePlugins does this)
     expect(getToolOwner("disable-tool")?.kind).toBe("plugin");
 
     writeFileSync(join(dir, ".disabled"), "");

package/src/agent/loop.ts

@@ -240,7 +240,15 @@ export type AgentEvent =
   | { type: "llm_call_started"; callSite?: LLMCallSite }
   | { type: "text_delta"; text: string }
   | { type: "thinking_delta"; thinking: string }
-  | { type: "message_complete"; message: Message }
+  /**
+   * Emitted once per LLM call when the assistant message is finalized.
+   * `model` is the provider-reported model that served the call
+   * (`response.model`) — the same value `llm_usage` records — so downstream
+   * persistence can attribute the message to the model that actually ran,
+   * including per-call reroutes by a `pre-model-call` hook. Absent on
+   * synthesized emissions that have no provider response.
+   */
+  | { type: "message_complete"; message: Message; model?: string }
   | { type: "max_tokens_reached"; stopReason: string }
   | {
       type: "tool_use";
@@ -1902,6 +1910,7 @@ export class AgentLoop {
             await onEvent({
               type: "message_complete",
               message: safeAssistantMessage,
+              model: response.model,
             });
             history = maxTokensMessages;
             continue;
@@ -1914,6 +1923,7 @@ export class AgentLoop {
           await onEvent({
             type: "message_complete",
             message: safeAssistantMessage,
+            model: response.model,
           });
           await stopTurn("max_tokens_reached");
           break;
@@ -1998,7 +2008,11 @@ export class AgentLoop {
 
         history.push(assistantMessage);
 
-        await onEvent({ type: "message_complete", message: assistantMessage });
+        await onEvent({
+          type: "message_complete",
+          message: assistantMessage,
+          model: response.model,
+        });
 
         if (toolUseBlocks.length === 0 || !this.toolExecutor) {
           // The model stopped requesting tools and `post-model-call` settled on

package/src/daemon/conversation-agent-loop-handlers.ts

@@ -589,12 +589,20 @@ async function persistLoopMessageContent(
   contentJson: string,
   op: string,
   rlog: pino.Logger,
+  metadataUpdates?: Record<string, unknown>,
 ): Promise<boolean> {
   try {
-    await withSqliteRetry(() => updateMessageContent(messageId, contentJson), {
-      op,
-      context: { messageId },
-    });
+    // Metadata updates (e.g. the served model at finalize) ride the same
+    // write as the content — `updateMessageContent` commits both atomically —
+    // so a partial write can't leave them out of sync; the updates
+    // shallow-merge onto the channel provenance stamped at reserve.
+    await withSqliteRetry(
+      () => updateMessageContent(messageId, contentJson, metadataUpdates),
+      {
+        op,
+        context: { messageId },
+      },
+    );
     return true;
   } catch (err) {
     rlog.error(
@@ -2074,11 +2082,17 @@ export async function handleMessageComplete(
     );
   }
   const contentJson = JSON.stringify(contentForPersistence);
+  // Stamp the served model carried on the event (`response.model`, the same
+  // value `llm_usage` records) onto the row alongside the content, so turn-trace
+  // assembly can attribute each assistant message to the model that actually
+  // ran it — including per-call reroutes by a `pre-model-call` hook. Absent on
+  // synthesized completions with no provider response; the key is omitted then.
   const persisted = await persistLoopMessageContent(
     assistantMessageId,
     contentJson,
     "finalize_assistant_message",
     deps.rlog,
+    event.model ? { model: event.model } : undefined,
   );
   state.assistantRowAwaitingFinalization = false;
   // The assistant row now holds the authoritative content (text + thinking +

package/src/daemon/conversation-tool-setup.ts

@@ -30,6 +30,7 @@ import {
   getToolOwner,
   getWorkspaceToolDefinitions,
   getWorkspaceToolNames,
+  loadPluginTools,
 } from "../tools/registry.js";
 import {
   ACTIVITY_SKIP_SET,
@@ -834,11 +835,15 @@ export function createResolveToolsCallback(
     // serialized, so the registry settles for a subsequent turn to read.
     void loadWorkspaceTools();
 
-    // Re-read plugin tool definitions from the registry each turn so a plugin
-    // installed/removed at runtime (activated by the per-turn scan in
-    // `plugins/mtime-cache.ts`) is picked up without recreating the
-    // conversation. Plugin tools share core's context filter + allowlist path,
-    // so combine them with the core snapshot before filtering.
+    // Same treatment for user-plugin tools: pull the plugin mtime-cache's
+    // active tool set into the registry (a no-op costs one sentinel stat +
+    // fingerprint compares), so a plugin installed/removed/edited at runtime
+    // is picked up without recreating the conversation.
+    void loadPluginTools();
+
+    // Re-read plugin tool definitions from the registry each turn. Plugin
+    // tools share core's context filter + allowlist path, so combine them
+    // with the core snapshot before filtering.
     const currentPluginDefs = getPluginToolDefinitions();
 
     // Scope plugin tools to the conversation's per-chat plugin set. `null`

package/src/daemon/external-plugins-bootstrap.ts

@@ -73,6 +73,7 @@ import {
   unregisterSkillRoute,
 } from "../runtime/skill-route-registry.js";
 import {
+  loadPluginTools,
   registerPluginTools,
   unregisterPluginTools,
 } from "../tools/registry.js";
@@ -152,6 +153,12 @@ function ensurePluginStorageDir(pluginName: string): string {
 export async function initializePlugins(): Promise<void> {
   registerDefaultPlugins();
   await loadUserPlugins();
+  // Pull the user-plugin tool set the cache scan above discovered into the
+  // tool registry. Kept in the same slot registration used to occupy (after
+  // the user-plugin scan, before the default plugins' `init()` registers
+  // their Plugin-object tools) so name-conflict precedence is unchanged.
+  // `loadPluginTools` never rejects — a failed pull logs and boots degraded.
+  await loadPluginTools();
   try {
     await bootstrapPlugins();
   } catch (err) {