@vellumai/assistant 0.5.1 → 0.5.3

package/src/__tests__/llm-context-route-provider.test.ts

@@ -0,0 +1,217 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+const testDir = mkdtempSync(join(tmpdir(), "llm-context-route-provider-"));
+
+mock.module("../util/platform.js", () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+import { getDb, initializeDb, resetDb } from "../memory/db.js";
+import { llmRequestLogs } from "../memory/schema.js";
+import { conversationQueryRouteDefinitions } from "../runtime/routes/conversation-query-routes.js";
+
+initializeDb();
+
+afterAll(() => {
+  resetDb();
+  try {
+    rmSync(testDir, { recursive: true });
+  } catch {
+    /* best effort */
+  }
+});
+
+const routes = conversationQueryRouteDefinitions();
+
+function dispatchLlmContext(messageId: string): Promise<Response> | Response {
+  const url = new URL(`http://localhost/v1/messages/${messageId}/llm-context`);
+  const route = routes.find(
+    (r) => r.method === "GET" && r.endpoint === "messages/:id/llm-context",
+  );
+  if (!route) {
+    throw new Error("No llm-context route found");
+  }
+
+  return route.handler({
+    req: new Request(url.toString(), { method: "GET" }),
+    url,
+    server: null as never,
+    authContext: {} as never,
+    params: { id: messageId },
+  });
+}
+
+function clearRequestLogs(): void {
+  getDb().delete(llmRequestLogs).run();
+}
+
+function seedRequestLog(overrides: {
+  id: string;
+  messageId: string;
+  provider: string | null;
+  requestPayload: string;
+  responsePayload: string;
+  createdAt?: number;
+}): void {
+  getDb()
+    .insert(llmRequestLogs)
+    .values({
+      id: overrides.id,
+      conversationId: "conv-1",
+      messageId: overrides.messageId,
+      provider: overrides.provider,
+      requestPayload: overrides.requestPayload,
+      responsePayload: overrides.responsePayload,
+      createdAt: overrides.createdAt ?? 1_700_000_000_000,
+    })
+    .run();
+}
+
+beforeEach(() => {
+  clearRequestLogs();
+});
+
+describe("GET /v1/messages/:id/llm-context provider preference", () => {
+  const openAiRequestPayload = JSON.stringify({
+    model: "gpt-4.1",
+    tool_choice: "auto",
+    messages: [
+      { role: "system", content: "Stay brief." },
+      { role: "user", content: "Hello there." },
+    ],
+  });
+
+  const openAiResponsePayload = JSON.stringify({
+    model: "gpt-4.1-2026-03-01",
+    choices: [
+      {
+        finish_reason: "stop",
+        message: {
+          role: "assistant",
+          content: "Hello back.",
+        },
+      },
+    ],
+    usage: {
+      prompt_tokens: 11,
+      completion_tokens: 4,
+    },
+  });
+
+  test("prefers a stored OpenRouter provider over OpenAI-shaped payload inference", async () => {
+    seedRequestLog({
+      id: "log-openrouter",
+      messageId: "msg-openrouter",
+      provider: "openrouter",
+      requestPayload: openAiRequestPayload,
+      responsePayload: openAiResponsePayload,
+    });
+
+    const response = await dispatchLlmContext("msg-openrouter");
+    expect(response.status).toBe(200);
+
+    const body = (await response.json()) as {
+      logs: Array<{
+        summary?: { provider: string };
+      }>;
+    };
+
+    expect(body.logs).toHaveLength(1);
+    expect(body.logs[0]?.summary).toEqual(
+      expect.objectContaining({
+        provider: "openrouter",
+      }),
+    );
+  });
+
+  test("prefers a stored Fireworks provider over OpenAI-shaped payload inference", async () => {
+    seedRequestLog({
+      id: "log-fireworks",
+      messageId: "msg-fireworks",
+      provider: "fireworks",
+      requestPayload: openAiRequestPayload,
+      responsePayload: openAiResponsePayload,
+    });
+
+    const response = await dispatchLlmContext("msg-fireworks");
+    expect(response.status).toBe(200);
+
+    const body = (await response.json()) as {
+      logs: Array<{
+        summary?: { provider: string };
+      }>;
+    };
+
+    expect(body.logs).toHaveLength(1);
+    expect(body.logs[0]?.summary).toEqual(
+      expect.objectContaining({
+        provider: "fireworks",
+      }),
+    );
+  });
+
+  test("keeps the legacy shape-inferred provider when no stored provider exists", async () => {
+    seedRequestLog({
+      id: "log-legacy",
+      messageId: "msg-legacy",
+      provider: null,
+      requestPayload: openAiRequestPayload,
+      responsePayload: openAiResponsePayload,
+    });
+
+    const response = await dispatchLlmContext("msg-legacy");
+    expect(response.status).toBe(200);
+
+    const body = (await response.json()) as {
+      logs: Array<{
+        summary?: { provider: string };
+      }>;
+    };
+
+    expect(body.logs).toHaveLength(1);
+    expect(body.logs[0]?.summary).toEqual(
+      expect.objectContaining({
+        provider: "openai",
+      }),
+    );
+  });
+
+  test("keeps a stored provider label even when payload normalization fails", async () => {
+    seedRequestLog({
+      id: "log-raw-only",
+      messageId: "msg-raw-only",
+      provider: "ollama",
+      requestPayload: "not-json",
+      responsePayload: "still-not-json",
+    });
+
+    const response = await dispatchLlmContext("msg-raw-only");
+    expect(response.status).toBe(200);
+
+    const body = (await response.json()) as {
+      logs: Array<{
+        summary?: { provider: string };
+      }>;
+    };
+
+    expect(body.logs).toHaveLength(1);
+    expect(body.logs[0]?.summary).toEqual({ provider: "ollama" });
+  });
+});

package/src/__tests__/llm-request-log-turn-query.test.ts

@@ -0,0 +1,270 @@
+import { mkdtempSync, realpathSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+const testDir = realpathSync(
+  mkdtempSync(join(tmpdir(), "llm-request-log-turn-query-test-")),
+);
+const workspaceDir = join(testDir, ".vellum", "workspace");
+const conversationsDir = join(workspaceDir, "conversations");
+
+mock.module("../util/platform.js", () => ({
+  getRootDir: () => join(testDir, ".vellum"),
+  getDataDir: () => join(workspaceDir, "data"),
+  getWorkspaceDir: () => workspaceDir,
+  getConversationsDir: () => conversationsDir,
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    ui: {},
+    model: "test",
+    provider: "test",
+    memory: { enabled: false },
+    rateLimit: { maxRequestsPerMinute: 0 },
+    secretDetection: { enabled: false },
+  }),
+}));
+
+import {
+  addMessage,
+  createConversation,
+  forkConversation,
+} from "../memory/conversation-crud.js";
+import { getDb, initializeDb, resetDb } from "../memory/db.js";
+import {
+  backfillMessageIdOnLogs,
+  getRequestLogsByMessageId,
+  recordRequestLog,
+} from "../memory/llm-request-log-store.js";
+import { llmRequestLogs, toolInvocations } from "../memory/schema.js";
+
+initializeDb();
+
+function resetTables(): void {
+  const db = getDb();
+  db.delete(llmRequestLogs).run();
+  db.delete(toolInvocations).run();
+  db.run("DELETE FROM message_attachments");
+  db.run("DELETE FROM attachments");
+  db.run("DELETE FROM messages");
+  db.run("DELETE FROM conversations");
+}
+
+function toolResultContent(toolUseIds: string[]): string {
+  return JSON.stringify(
+    toolUseIds.map((id) => ({
+      type: "tool_result",
+      tool_use_id: id,
+      content: "ok",
+      is_error: false,
+    })),
+  );
+}
+
+afterAll(() => {
+  resetDb();
+  try {
+    rmSync(testDir, { recursive: true, force: true });
+  } catch {
+    /* best effort */
+  }
+});
+
+describe("getRequestLogsByMessageId — turn-aware query", () => {
+  beforeEach(() => {
+    resetTables();
+  });
+
+  test("single message, single log: backward compat — returns 1 log", async () => {
+    const conv = createConversation("single-msg");
+    await addMessage(conv.id, "user", "Hello", undefined, {
+      skipIndexing: true,
+    });
+    const a1 = await addMessage(conv.id, "assistant", "Hi!", undefined, {
+      skipIndexing: true,
+    });
+
+    // Record a log without messageId, then backfill
+    recordRequestLog(conv.id, '{"prompt":"hi"}', '{"result":"hello"}');
+    backfillMessageIdOnLogs(conv.id, a1.id);
+
+    const logs = getRequestLogsByMessageId(a1.id);
+    expect(logs).toHaveLength(1);
+    expect(logs[0]?.messageId).toBe(a1.id);
+    expect(logs[0]?.conversationId).toBe(conv.id);
+  });
+
+  test("multi-step turn: returns logs from all assistant messages in the turn", async () => {
+    const conv = createConversation("multi-step");
+
+    // user → A1 (+ log1) → tool_result → A2 (+ log2)
+    await addMessage(conv.id, "user", "Do the task", undefined, {
+      skipIndexing: true,
+    });
+
+    // First LLM call → A1
+    recordRequestLog(conv.id, '{"step":1}', '{"tool_use":"bash"}');
+    const a1 = await addMessage(
+      conv.id,
+      "assistant",
+      "Using tool...",
+      undefined,
+      { skipIndexing: true },
+    );
+    backfillMessageIdOnLogs(conv.id, a1.id);
+
+    // tool_result user message
+    await addMessage(
+      conv.id,
+      "user",
+      toolResultContent(["tool-1"]),
+      undefined,
+      { skipIndexing: true },
+    );
+
+    // Second LLM call → A2
+    recordRequestLog(conv.id, '{"step":2}', '{"result":"done"}');
+    const a2 = await addMessage(conv.id, "assistant", "All done!", undefined, {
+      skipIndexing: true,
+    });
+    backfillMessageIdOnLogs(conv.id, a2.id);
+
+    // Query from A2 (the last message in the turn) → should return both logs
+    const logs = getRequestLogsByMessageId(a2.id);
+    expect(logs).toHaveLength(2);
+    expect(logs[0]?.messageId).toBe(a1.id);
+    expect(logs[1]?.messageId).toBe(a2.id);
+    // Verify ordering is by createdAt ASC
+    expect(logs[0]!.createdAt).toBeLessThanOrEqual(logs[1]!.createdAt);
+  });
+
+  test("fork fallback still works: forked message with no logs, source has turn logs", async () => {
+    const source = createConversation("source-conv");
+
+    // Build a multi-step turn in the source conversation
+    await addMessage(source.id, "user", "Original task", undefined, {
+      skipIndexing: true,
+    });
+
+    recordRequestLog(source.id, '{"step":1}', '{"tool":"bash"}');
+    const a1 = await addMessage(
+      source.id,
+      "assistant",
+      "Using tool...",
+      undefined,
+      { skipIndexing: true },
+    );
+    backfillMessageIdOnLogs(source.id, a1.id);
+
+    await addMessage(
+      source.id,
+      "user",
+      toolResultContent(["tool-1"]),
+      undefined,
+      { skipIndexing: true },
+    );
+
+    recordRequestLog(source.id, '{"step":2}', '{"result":"ok"}');
+    const a2 = await addMessage(
+      source.id,
+      "assistant",
+      "Done with source!",
+      undefined,
+      { skipIndexing: true },
+    );
+    backfillMessageIdOnLogs(source.id, a2.id);
+
+    // Fork the conversation
+    const fork = forkConversation({ conversationId: source.id });
+    const forkMessages = (
+      await import("../memory/conversation-crud.js")
+    ).getMessages(fork.id);
+    const forkLastAssistant = forkMessages
+      .filter((m) => m.role === "assistant")
+      .at(-1);
+    expect(forkLastAssistant).toBeDefined();
+
+    // The fork has no LLM logs of its own — should fall back to source turn's logs
+    const logs = getRequestLogsByMessageId(forkLastAssistant!.id);
+    expect(logs).toHaveLength(2);
+    expect(logs[0]?.conversationId).toBe(source.id);
+    expect(logs[1]?.conversationId).toBe(source.id);
+  });
+
+  test("logs from different turns don't bleed", async () => {
+    const conv = createConversation("two-turns");
+
+    // First turn: user → A1 (+ log1)
+    await addMessage(conv.id, "user", "First question", undefined, {
+      skipIndexing: true,
+    });
+    recordRequestLog(conv.id, '{"turn":1}', '{"answer":"first"}');
+    const a1 = await addMessage(
+      conv.id,
+      "assistant",
+      "First answer",
+      undefined,
+      { skipIndexing: true },
+    );
+    backfillMessageIdOnLogs(conv.id, a1.id);
+
+    // Second turn: user → A2 (+ log2) → tool_result → A3 (+ log3)
+    await addMessage(conv.id, "user", "Second question", undefined, {
+      skipIndexing: true,
+    });
+    recordRequestLog(conv.id, '{"turn":2,"step":1}', '{"tool":"bash"}');
+    const a2 = await addMessage(
+      conv.id,
+      "assistant",
+      "Using tool...",
+      undefined,
+      { skipIndexing: true },
+    );
+    backfillMessageIdOnLogs(conv.id, a2.id);
+
+    await addMessage(
+      conv.id,
+      "user",
+      toolResultContent(["tool-2"]),
+      undefined,
+      { skipIndexing: true },
+    );
+
+    recordRequestLog(conv.id, '{"turn":2,"step":2}', '{"result":"done"}');
+    const a3 = await addMessage(
+      conv.id,
+      "assistant",
+      "Second done!",
+      undefined,
+      { skipIndexing: true },
+    );
+    backfillMessageIdOnLogs(conv.id, a3.id);
+
+    // Query second turn → should only return logs for A2 and A3, NOT A1
+    const secondTurnLogs = getRequestLogsByMessageId(a3.id);
+    expect(secondTurnLogs).toHaveLength(2);
+    expect(secondTurnLogs[0]?.messageId).toBe(a2.id);
+    expect(secondTurnLogs[1]?.messageId).toBe(a3.id);
+
+    // Query first turn → should only return log for A1
+    const firstTurnLogs = getRequestLogsByMessageId(a1.id);
+    expect(firstTurnLogs).toHaveLength(1);
+    expect(firstTurnLogs[0]?.messageId).toBe(a1.id);
+  });
+});

package/src/__tests__/media-generate-image.test.ts

@@ -77,84 +77,6 @@ mock.module("../providers/managed-proxy/context.js", () => ({
   resolveManagedProxyContext: async () => mockManagedProxyContext,
 }));
 
-let mockAttachments: Array<{
-  id: string;
-  assistantId: string;
-  originalFilename: string;
-  mimeType: string;
-  sizeBytes: number;
-  kind: string;
-  createdAt: number;
-  dataBase64: string;
-}> = [];
-
-mock.module("../memory/attachments-store.js", () => ({
-  getAttachmentsByIds: (_assistantId: string, _ids: string[]) =>
-    mockAttachments,
-  getAttachmentContent: (id: string) => {
-    const att = mockAttachments.find((a) => a.id === id);
-    if (!att) return null;
-    return Buffer.from(att.dataBase64, "base64");
-  },
-}));
-
-mock.module("../memory/db.js", () => ({
-  getDb: () => ({
-    select: () => ({
-      from: () => ({
-        where: () => ({
-          get: () => ({ assistantId: "test-assistant" }),
-        }),
-      }),
-    }),
-  }),
-}));
-
-mock.module("../memory/schema.js", () => ({
-  conversationKeys: {
-    assistantId: "assistantId",
-    conversationId: "conversationId",
-  },
-}));
-
-mock.module("drizzle-orm", () => ({
-  eq: () => true,
-}));
-
-mock.module("../memory/conversation-crud.js", () => ({
-  setConversationOriginChannelIfUnset: () => {},
-  updateConversationContextWindow: () => {},
-  deleteMessageById: () => {},
-  updateConversationTitle: () => {},
-  updateConversationUsage: () => {},
-  addMessage: () => ({ id: "mock-msg-id" }),
-  getMessages: () => [],
-  getConversation: () => ({
-    id: "conv-1",
-    contextSummary: null,
-    contextCompactedMessageCount: 0,
-    totalInputTokens: 0,
-    totalOutputTokens: 0,
-    totalEstimatedCost: 0,
-    title: null,
-  }),
-  provenanceFromTrustContext: () => ({
-    source: "user",
-    trustContext: undefined,
-  }),
-  getConversationOriginInterface: () => null,
-  getConversationOriginChannel: () => null,
-  getConversationType: () => "standard",
-}));
-
-mock.module("../daemon/media-visibility-policy.js", () => ({
-  isAttachmentVisible: () => true,
-}));
-
-mock.module("../tools/assets/search.js", () => ({
-  getAttachmentSourceConversations: () => [],
-}));
-
 // Import after mocking
 import { run } from "../config/bundled-skills/image-studio/tools/media-generate-image.js";
 
@@ -183,7 +105,6 @@ beforeEach(() => {
     resolvedModel: "gemini-3.1-flash-image-preview",
   };
   mockGenerateError = null;
-  mockAttachments = [];
   lastGenerateCredentials = null;
   mockManagedBaseUrl = undefined;
   mockManagedProxyContext = {
@@ -307,26 +228,57 @@ describe("image-studio skill script wrapper", () => {
     expect(result.content).toContain("Mock error: API failure");
   });
 
-  test("passes attachment data as sourceImages for edit mode", async () => {
-    mockAttachments = [
+  test("reads source images from file paths on disk", async () => {
+    // Write a temp image file inside the workspace (fakeContext.workingDir = /tmp)
+    const tmpPath = join("/tmp", "test-source-image.png");
+    const pngBytes = Buffer.from(
+      "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==",
+      "base64",
+    );
+    await Bun.write(tmpPath, pngBytes);
+
+    try {
+      const result = await run(
+        { prompt: "edit this", mode: "edit", source_paths: [tmpPath] },
+        fakeContext,
+      );
+
+      expect(result.isError).toBe(false);
+      expect(result.content).toContain("Generated 1 image");
+    } finally {
+      const { unlink } = await import("fs/promises");
+      if (await Bun.file(tmpPath).exists()) await unlink(tmpPath);
+    }
+  });
+
+  test("returns error when all source_paths are invalid", async () => {
+    const result = await run(
       {
-        id: "att-1",
-        assistantId: "test-assistant",
-        originalFilename: "photo.jpg",
-        mimeType: "image/jpeg",
-        sizeBytes: 1000,
-        kind: "image",
-        createdAt: Date.now(),
-        dataBase64: "attachment-data",
+        prompt: "edit this",
+        mode: "edit",
+        source_paths: ["/nonexistent/path.png"],
       },
-    ];
+      fakeContext,
+    );
 
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain(
+      "None of the specified file paths could be read",
+    );
+  });
+
+  test("rejects source_paths outside the workspace", async () => {
     const result = await run(
-      { prompt: "remove bg", mode: "edit", attachment_ids: ["att-1"] },
+      {
+        prompt: "edit this",
+        mode: "edit",
+        source_paths: ["/etc/passwd"],
+      },
       fakeContext,
     );
 
-    expect(result.isError).toBe(false);
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("outside the working directory");
   });
 });
 
@@ -370,10 +322,11 @@ describe("image-studio TOOLS.json manifest", () => {
     expect(schema.properties.prompt.type).toBe("string");
   });
 
-  test("input schema has optional mode, attachment_ids, model, variants", () => {
+  test("input schema has optional mode, source_paths, model, variants", () => {
     const props = manifest.tools[0].input_schema.properties;
     expect(props.mode.enum).toEqual(["generate", "edit"]);
-    expect(props.attachment_ids.type).toBe("array");
+    expect(props.source_paths.type).toBe("array");
+    expect(props.attachment_ids).toBeUndefined();
     expect(props.model.enum).toEqual([
       "gemini-3.1-flash-image-preview",
       "gemini-3-pro-image-preview",