@vellumai/assistant 0.8.3 → 0.8.5

package/src/runtime/routes/__tests__/content-source-routes.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Unit tests for content-source-routes.ts.
+ *
+ * Drives the handler function directly (bypassing the router) and mocks
+ * out node:fs writes so no real I/O occurs.
+ */
+
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ---------------------------------------------------------------------------
+// Module mocks — must appear before any imports of the module under test
+// ---------------------------------------------------------------------------
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+const FAKE_WORKSPACE = "/tmp/content-source-routes-test-workspace";
+
+mock.module("../../../util/platform.js", () => ({
+  getWorkspaceDir: () => FAKE_WORKSPACE,
+}));
+
+const writtenFiles = new Map<string, string>();
+
+mock.module("node:fs", () => ({
+  mkdirSync: () => {},
+  writeFileSync: (path: string, content: string) => {
+    writtenFiles.set(path, content);
+  },
+}));
+
+// ---------------------------------------------------------------------------
+// Imports after mocks
+// ---------------------------------------------------------------------------
+
+import { ROUTES } from "../content-source-routes.js";
+import type { RouteDefinition, RouteHandlerArgs } from "../types.js";
+
+afterAll(() => {
+  mock.restore();
+});
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function findHandler(operationId: string): RouteDefinition["handler"] {
+  const route = ROUTES.find((r) => r.operationId === operationId);
+  if (!route) throw new Error(`Route ${operationId} not found`);
+  return route.handler;
+}
+
+function makeArgs(body?: Record<string, unknown>): RouteHandlerArgs {
+  return { body };
+}
+
+const handler = findHandler("content_source_set");
+
+beforeEach(() => {
+  writtenFiles.clear();
+});
+
+// ---------------------------------------------------------------------------
+// URL validation
+// ---------------------------------------------------------------------------
+
+describe("content_source_set — URL validation", () => {
+  test("https URL is accepted and sidecar written", () => {
+    const result = handler(makeArgs({ url: "https://myblog.com/posts" }));
+    expect(result).toEqual({ ok: true });
+
+    const expectedPath = `${FAKE_WORKSPACE}/data/content-source.json`;
+    expect(writtenFiles.has(expectedPath)).toBe(true);
+    const written = JSON.parse(writtenFiles.get(expectedPath)!);
+    expect(written.url).toBe("https://myblog.com/posts");
+  });
+
+  test("http URL is accepted and sidecar written", () => {
+    const result = handler(makeArgs({ url: "http://intranet.example.com" }));
+    expect(result).toEqual({ ok: true });
+  });
+
+  test("URL with leading/trailing whitespace is trimmed", () => {
+    const result = handler(makeArgs({ url: "  https://blog.example.com  " }));
+    expect(result).toEqual({ ok: true });
+
+    const expectedPath = `${FAKE_WORKSPACE}/data/content-source.json`;
+    const written = JSON.parse(writtenFiles.get(expectedPath)!);
+    expect(written.url).toBe("https://blog.example.com/");
+  });
+
+  test("bare hostname without protocol returns invalid_url", () => {
+    const result = handler(makeArgs({ url: "myblog.com" }));
+    expect(result).toEqual({ ok: false, error: "invalid_url" });
+    expect(writtenFiles.size).toBe(0);
+  });
+
+  test("ftp:// URL is rejected", () => {
+    const result = handler(makeArgs({ url: "ftp://files.example.com" }));
+    expect(result).toEqual({ ok: false, error: "invalid_url" });
+    expect(writtenFiles.size).toBe(0);
+  });
+
+  test("empty string returns invalid_url", () => {
+    const result = handler(makeArgs({ url: "" }));
+    expect(result).toEqual({ ok: false, error: "invalid_url" });
+    expect(writtenFiles.size).toBe(0);
+  });
+
+  test("whitespace-only string returns invalid_url", () => {
+    const result = handler(makeArgs({ url: "   " }));
+    expect(result).toEqual({ ok: false, error: "invalid_url" });
+    expect(writtenFiles.size).toBe(0);
+  });
+
+  test("javascript: URL is rejected", () => {
+    const result = handler(makeArgs({ url: "javascript:alert(1)" }));
+    expect(result).toEqual({ ok: false, error: "invalid_url" });
+    expect(writtenFiles.size).toBe(0);
+  });
+
+  test("missing url field returns invalid_url", () => {
+    const result = handler(makeArgs({}));
+    expect(result).toEqual({ ok: false, error: "invalid_url" });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Sidecar content verification
+// ---------------------------------------------------------------------------
+
+describe("content_source_set — sidecar contents", () => {
+  test("writes url to data/content-source.json", () => {
+    handler(makeArgs({ url: "https://example.com/blog" }));
+
+    const expectedPath = `${FAKE_WORKSPACE}/data/content-source.json`;
+    expect(writtenFiles.has(expectedPath)).toBe(true);
+
+    const written = JSON.parse(writtenFiles.get(expectedPath)!);
+    expect(Object.keys(written)).toEqual(["url"]);
+  });
+
+  test("no sidecar written on invalid URL", () => {
+    handler(makeArgs({ url: "not-a-url" }));
+    expect(writtenFiles.size).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Policy key verification
+// ---------------------------------------------------------------------------
+
+describe("route policy key", () => {
+  test("content_source_set uses policyKey: secrets", () => {
+    const route = ROUTES.find((r) => r.operationId === "content_source_set");
+    expect(route?.policyKey).toBe("secrets");
+  });
+});

package/src/runtime/routes/__tests__/conversation-query-routes.test.ts

@@ -66,6 +66,7 @@ import {
   recordMemoryV2ActivationLog,
 } from "../../../memory/memory-v2-activation-log-store.js";
 import {
+  conversationKeys,
   conversations,
   llmRequestLogs,
   memoryV2ActivationLogs,
@@ -89,6 +90,10 @@ const llmContextRoute = ROUTES.find(
   (r) => r.method === "GET" && r.endpoint === "messages/:id/llm-context",
 )!;
 
+const conversationLlmContextRoute = ROUTES.find(
+  (r) => r.method === "GET" && r.endpoint === "conversations/llm-context",
+)!;
+
 const replaceProfileRoute = ROUTES.find(
   (r) => r.operationId === "config_llm_profiles_replace",
 )!;
@@ -97,11 +102,16 @@ function dispatchLlmContext(messageId: string) {
   return llmContextRoute.handler({ pathParams: { id: messageId } });
 }
 
+function dispatchConversationLlmContext(queryParams: Record<string, string>) {
+  return conversationLlmContextRoute.handler({ queryParams });
+}
+
 function clearTables(): void {
   const db = getDb();
   db.delete(llmRequestLogs).run();
   db.delete(memoryV2ActivationLogs).run();
   db.delete(messages).run();
+  db.delete(conversationKeys).run();
   db.delete(conversations).run();
 }
 
@@ -141,7 +151,11 @@ function seedConversationAndMessage(args: {
     .run();
 }
 
-function seedRequestLog(messageId: string, id: string): void {
+function seedRequestLog(
+  messageId: string,
+  id: string,
+  options: { agentLoopExitReason?: string | null } = {},
+): void {
   getDb()
     .insert(llmRequestLogs)
     .values({
@@ -154,10 +168,109 @@ function seedRequestLog(messageId: string, id: string): void {
         choices: [{ message: { content: "hi" } }],
       }),
       createdAt: 1_700_000_000_000,
+      ...(options.agentLoopExitReason != null
+        ? { agentLoopExitReason: options.agentLoopExitReason }
+        : {}),
+    })
+    .run();
+}
+
+function seedConversationKey(conversationKey: string, conversationId: string): void {
+  getDb()
+    .insert(conversationKeys)
+    .values({
+      id: `key-${conversationKey}`,
+      conversationKey,
+      conversationId,
+      createdAt: Date.now(),
     })
     .run();
 }
 
+describe("GET /v1/conversations/llm-context", () => {
+  beforeEach(() => {
+    clearTables();
+  });
+
+  test("returns all LLM calls for a resolved conversation key", async () => {
+    seedConversationAndMessage({
+      conversationId: "conv-1",
+      messageId: "msg-1",
+      source: "user",
+      conversationType: "standard",
+      totalEstimatedCost: 0.42,
+    });
+    getDb()
+      .insert(messages)
+      .values({
+        id: "msg-2",
+        conversationId: "conv-1",
+        role: "assistant",
+        content: "",
+        createdAt: Date.now() + 1,
+        metadata: null,
+      })
+      .run();
+    seedConversationAndMessage({
+      conversationId: "conv-other",
+      messageId: "msg-other",
+      source: "user",
+      conversationType: "standard",
+    });
+    seedConversationKey("conv-key", "conv-1");
+    seedRequestLog("msg-2", "log-b");
+    seedRequestLog("msg-1", "log-a");
+    getDb()
+      .insert(llmRequestLogs)
+      .values({
+        id: "log-other",
+        conversationId: "conv-other",
+        messageId: "msg-other",
+        provider: "openai",
+        requestPayload: JSON.stringify({ model: "gpt-4.1", messages: [] }),
+        responsePayload: JSON.stringify({
+          choices: [{ message: { content: "other" } }],
+        }),
+        createdAt: 1_700_000_000_001,
+      })
+      .run();
+
+    const body = (await dispatchConversationLlmContext({
+      conversationKey: "conv-key",
+    })) as {
+      conversationId: string;
+      conversationKey: string;
+      conversationKind: string;
+      conversationTotalEstimatedCostUsd: number | null;
+      logs: Array<{ id: string }>;
+      memoryRecall: null;
+      memoryV2Activation: null;
+    };
+
+    expect(body.conversationId).toBe("conv-1");
+    expect(body.conversationKey).toBe("conv-key");
+    expect(body.conversationKind).toBe("user");
+    expect(body.conversationTotalEstimatedCostUsd).toBe(0.42);
+    expect(body.logs.map((log) => log.id)).toEqual(["log-a", "log-b"]);
+    expect(body.memoryRecall).toBeNull();
+    expect(body.memoryV2Activation).toBeNull();
+  });
+
+  test("returns an empty inspector response for an unresolved conversation key", async () => {
+    const body = (await dispatchConversationLlmContext({
+      conversationKey: "missing-key",
+    })) as {
+      conversationId: string | null;
+      conversationKey: string;
+      logs: unknown[];
+    };
+
+    expect(body.conversationId).toBeNull();
+    expect(body.conversationKey).toBe("missing-key");
+    expect(body.logs).toEqual([]);
+  });
+});
+
 describe("GET /v1/messages/:id/llm-context — memoryV2Activation", () => {
   beforeEach(() => {
     clearTables();
@@ -317,6 +430,53 @@ describe("GET /v1/messages/:id/llm-context — conversationTotalEstimatedCostUsd
   });
 });
 
+describe("GET /v1/messages/:id/llm-context — agentLoopExitReason", () => {
+  beforeEach(() => {
+    clearTables();
+  });
+
+  test("surfaces the stamped agent_loop_exit_reason on the terminal log", async () => {
+    const messageId = "msg-with-exit";
+    seedConversationAndMessage({
+      conversationId: "conv-1",
+      messageId,
+      source: "user",
+      conversationType: "standard",
+    });
+    // Two logs in the same turn — only the terminal one is stamped.
+    seedRequestLog(messageId, "log-non-terminal");
+    seedRequestLog(messageId, "log-terminal", {
+      agentLoopExitReason: "no_tool_calls",
+    });
+
+    const body = (await dispatchLlmContext(messageId)) as {
+      logs: Array<{ id: string; agentLoopExitReason: string | null }>;
+    };
+
+    const byId = new Map(body.logs.map((l) => [l.id, l.agentLoopExitReason]));
+    expect(byId.get("log-non-terminal")).toBeNull();
+    expect(byId.get("log-terminal")).toBe("no_tool_calls");
+  });
+
+  test("returns null when no log in the turn has been stamped", async () => {
+    const messageId = "msg-no-exit";
+    seedConversationAndMessage({
+      conversationId: "conv-1",
+      messageId,
+      source: "user",
+      conversationType: "standard",
+    });
+    seedRequestLog(messageId, "log-unstamped");
+
+    const body = (await dispatchLlmContext(messageId)) as {
+      logs: Array<{ id: string; agentLoopExitReason: string | null }>;
+    };
+
+    expect(body.logs).toHaveLength(1);
+    expect(body.logs[0]!.agentLoopExitReason).toBeNull();
+  });
+});
+
 describe("PUT /v1/config/llm/profiles/:name", () => {
   beforeEach(() => {
     savedRawConfig = null;

package/src/runtime/routes/__tests__/memory-v2-routes.test.ts

@@ -145,3 +145,17 @@ describe("memory_v2_list_concept_pages handler", () => {
     expect(result.pages.map((p) => p.slug)).toEqual(["valid-page"]);
   });
 });
+
+describe("memory_v2_ema_scores route registration", () => {
+  test("route is registered with the expected operationId and POST endpoint", () => {
+    const route = ROUTES.find((r) => r.operationId === "memory_v2_ema_scores");
+    expect(route).toBeDefined();
+    expect(route!.method).toBe("POST");
+    expect(route!.endpoint).toBe("memory/v2/ema-scores");
+    expect(route!.tags).toContain("memory");
+    // Schema rejects unknown keys so accidental request-body fields fail
+    // loudly during route adapter parsing rather than silently propagating.
+    expect(() => route!.requestBody!.parse({ extra: true })).toThrow();
+    expect(() => route!.requestBody!.parse({})).not.toThrow();
+  });
+});

package/src/runtime/routes/__tests__/memory-v2-simulate-route.test.ts

@@ -0,0 +1,290 @@
+/**
+ * Tests for the `memory_v2_simulate_router` route handler.
+ *
+ * The route is a read-only playground for previewing router config knob
+ * changes (`tier1_size`, `tier2_size`, `batch_size`) against the live page
+ * index. Tests assert:
+ *   1. The handler returns selected slugs with `sourceBySlug` populated.
+ *   2. Overrides are reflected in the response's `effectiveConfig`.
+ *   3. The handler never calls `recordInjectionEvents` — the simulate path
+ *      must not touch the EMA event log.
+ *
+ * Workspace lives in a `mkdtemp` directory per test; `~/.vellum/` is never
+ * touched. The provider and DB are stubbed so no network or SQLite I/O
+ * fires.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+  ToolUseContent,
+} from "../../../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Mocks (installed before the route module is imported)
+// ---------------------------------------------------------------------------
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Skill store: empty by default so the page index only contains test pages.
+mock.module("../../../memory/v2/skill-store.js", () => ({
+  SKILL_SLUG_PREFIX: "skills/",
+  listSkillEntries: () => [],
+  seedV2SkillEntries: async () => undefined,
+}));
+
+// NOW.md loader: return a fixed string. The route reads NOW from the
+// workspace at call time; stubbing keeps the test independent of disk state.
+mock.module("../../../memory/v2/now-text.js", () => ({
+  loadNowText: async () => "2026-05-22 14:00 PT",
+}));
+
+// Injection-events module. `recordInjectionEvents` must never be called by
+// the simulate path — record any call so the assertion catches a regression.
+// `computeInjectionScores` is called twice per simulate (once inside runRouter
+// for tier 2, once in the handler for the response payload); always returns
+// zero scores since the test workspace has no event history.
+const recordCalls: Array<{ slugs: readonly string[]; at: number }> = [];
+mock.module("../../../memory/v2/injection-events.js", () => ({
+  recordInjectionEvents: (
+    _db: unknown,
+    slugs: readonly string[],
+    at: number,
+  ) => {
+    recordCalls.push({ slugs, at });
+  },
+  computeInjectionScores: (
+    _db: unknown,
+    slugs: readonly string[],
+    _now: number,
+  ): Map<string, number> => new Map(slugs.map((s) => [s, 0])),
+}));
+
+// Database handle: the simulate route only passes this through to
+// `runRouter` and `computeInjectionScores`. Both are stubbed above, so a
+// sentinel object is sufficient.
+mock.module("../../../memory/db-connection.js", () => ({
+  getDb: () => ({ __stub: true }),
+}));
+
+// Config loader. The simulate route reads `memory.v2.enabled` (must be
+// true) and the full `memory.v2.router` block (overrides merged on top).
+const liveRouterConfig = {
+  enabled: true,
+  max_page_ids: 25,
+  router_prompt_path: null,
+  batch_size: null,
+  tier1_size: null,
+  tier2_size: null,
+};
+const mockConfigValue = {
+  memory: {
+    v2: {
+      enabled: true,
+      router: liveRouterConfig,
+    },
+  },
+};
+mock.module("../../../config/loader.js", () => ({
+  loadConfig: () => mockConfigValue,
+  getConfig: () => mockConfigValue,
+  getConfigReadOnly: () => mockConfigValue,
+  invalidateConfigCache: () => {},
+  API_KEY_PROVIDERS: [],
+}));
+
+// Provider stub. Default returns [1, 2] (selects first two pages).
+let providerStub: Provider | null = null;
+interface ProviderCall {
+  messages: Message[];
+  tools: ToolDefinition[] | undefined;
+  systemPrompt: string | undefined;
+  options: SendMessageOptions | undefined;
+}
+const providerCalls: ProviderCall[] = [];
+mock.module("../../../providers/provider-send-message.js", () => ({
+  getConfiguredProvider: async () => providerStub,
+  extractToolUse: (response: ProviderResponse) =>
+    response.content.find((b): b is ToolUseContent => b.type === "tool_use"),
+}));
+
+// Platform helpers. `getWorkspaceDir` must return the per-test tmp dir so
+// the route's page index points at the test workspace.
+let workspaceDir = "";
+mock.module("../../../util/platform.js", () => ({
+  getWorkspaceDir: () => workspaceDir,
+}));
+
+// ---------------------------------------------------------------------------
+// Import under test (after all mocks above)
+// ---------------------------------------------------------------------------
+
+const { handleSimulateRouter } = await import("../memory-v2-routes.js");
+const { writePage } = await import("../../../memory/v2/page-store.js");
+const { invalidatePageIndex } =
+  await import("../../../memory/v2/page-index.js");
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeProvider(pageIds: number[]): Provider {
+  return {
+    name: "stub",
+    sendMessage: async (messages, tools, systemPrompt, options) => {
+      providerCalls.push({ messages, tools, systemPrompt, options });
+      return {
+        model: "stub-model",
+        stopReason: "tool_use",
+        usage: { inputTokens: 0, outputTokens: 0 },
+        content: [
+          {
+            type: "tool_use",
+            id: "tu-1",
+            name: "select_pages_to_inject",
+            input: { page_ids: pageIds },
+          },
+        ],
+      };
+    },
+  };
+}
+
+function makePage(slug: string, summary: string) {
+  return {
+    slug,
+    frontmatter: {
+      edges: [],
+      ref_files: [],
+      ref_urls: [],
+      summary,
+    },
+    body: "",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Per-test setup / teardown
+// ---------------------------------------------------------------------------
+
+beforeEach(() => {
+  workspaceDir = mkdtempSync(join(tmpdir(), "memory-v2-simulate-test-"));
+  recordCalls.length = 0;
+  providerCalls.length = 0;
+  providerStub = null;
+  // Reset live router config to defaults between tests.
+  liveRouterConfig.batch_size = null;
+  liveRouterConfig.tier1_size = null;
+  liveRouterConfig.tier2_size = null;
+  liveRouterConfig.max_page_ids = 25;
+  invalidatePageIndex();
+});
+
+afterEach(() => {
+  invalidatePageIndex();
+  rmSync(workspaceDir, { recursive: true, force: true });
+});
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("handleSimulateRouter", () => {
+  test("returns selectedSlugs + sourceBySlug populated for each pick", async () => {
+    await writePage(workspaceDir, makePage("alice", "A"));
+    await writePage(workspaceDir, makePage("bob", "B"));
+    await writePage(workspaceDir, makePage("carol", "C"));
+    providerStub = makeProvider([3, 1]);
+
+    const result = await handleSimulateRouter({
+      body: {
+        recentTurnPairs: [
+          { assistantMessage: "", userMessage: "what's relevant?" },
+        ],
+      },
+    });
+
+    expect(result.failureReason).toBeNull();
+    expect(result.selectedSlugs).toEqual(["carol", "alice"]);
+    expect(result.sourceBySlug["carol"]).toBe("tier3:0");
+    expect(result.sourceBySlug["alice"]).toBe("tier3:0");
+    expect(result.scores["carol"]).toBe(0);
+    expect(result.scores["alice"]).toBe(0);
+    expect(result.totalCandidatePages).toBe(3);
+  });
+
+  test("propagates overrides into effectiveConfig and reports them in overrides", async () => {
+    await writePage(workspaceDir, makePage("alice", "A"));
+    providerStub = makeProvider([1]);
+
+    const result = await handleSimulateRouter({
+      body: {
+        recentTurnPairs: [{ assistantMessage: "", userMessage: "test" }],
+        configOverrides: {
+          tier1_size: 50,
+          batch_size: 25,
+        },
+      },
+    });
+
+    expect(result.effectiveConfig.tier1_size).toBe(50);
+    expect(result.effectiveConfig.tier2_size).toBeNull(); // not overridden, inherits live
+    expect(result.effectiveConfig.batch_size).toBe(25);
+    expect(result.effectiveConfig.max_page_ids).toBe(25);
+    expect(result.overrides).toEqual({ tier1_size: 50, batch_size: 25 });
+  });
+
+  test("never writes to the EMA event log (recordInjectionEvents not called)", async () => {
+    await writePage(workspaceDir, makePage("alice", "A"));
+    await writePage(workspaceDir, makePage("bob", "B"));
+    providerStub = makeProvider([1, 2]);
+
+    await handleSimulateRouter({
+      body: {
+        recentTurnPairs: [
+          { assistantMessage: "", userMessage: "should not record" },
+        ],
+      },
+    });
+
+    expect(recordCalls).toEqual([]);
+  });
+
+  test("rejects an empty last-pair userMessage at the schema layer", async () => {
+    await expect(
+      handleSimulateRouter({
+        body: { recentTurnPairs: [{ assistantMessage: "", userMessage: "" }] },
+      }),
+    ).rejects.toThrow();
+  });
+
+  test("rejects an empty recentTurnPairs array at the schema layer", async () => {
+    await expect(
+      handleSimulateRouter({ body: { recentTurnPairs: [] } }),
+    ).rejects.toThrow();
+  });
+
+  test("rejects negative tier size at the schema layer", async () => {
+    await expect(
+      handleSimulateRouter({
+        body: {
+          recentTurnPairs: [{ assistantMessage: "", userMessage: "test" }],
+          configOverrides: { tier1_size: -5 },
+        },
+      }),
+    ).rejects.toThrow();
+  });
+});