@vellumai/assistant 0.8.2 → 0.8.4

package/src/runtime/routes/__tests__/memory-v2-simulate-route.test.ts

@@ -0,0 +1,271 @@
+/**
+ * Tests for the `memory_v2_simulate_router` route handler.
+ *
+ * The route is a read-only playground for previewing router config knob
+ * changes (`tier1_size`, `tier2_size`, `batch_size`) against the live page
+ * index. Tests assert:
+ *   1. The handler returns selected slugs with `sourceBySlug` populated.
+ *   2. Overrides are reflected in the response's `effectiveConfig`.
+ *   3. The handler never calls `recordInjectionEvents` — the simulate path
+ *      must not touch the EMA event log.
+ *
+ * Workspace lives in a `mkdtemp` directory per test; `~/.vellum/` is never
+ * touched. The provider and DB are stubbed so no network or SQLite I/O
+ * fires.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+  ToolUseContent,
+} from "../../../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Mocks (installed before the route module is imported)
+// ---------------------------------------------------------------------------
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Skill store: empty by default so the page index only contains test pages.
+mock.module("../../../memory/v2/skill-store.js", () => ({
+  SKILL_SLUG_PREFIX: "skills/",
+  listSkillEntries: () => [],
+  seedV2SkillEntries: async () => undefined,
+}));
+
+// NOW.md loader: return a fixed string. The route reads NOW from the
+// workspace at call time; stubbing keeps the test independent of disk state.
+mock.module("../../../memory/v2/now-text.js", () => ({
+  loadNowText: async () => "2026-05-22 14:00 PT",
+}));
+
+// Injection-events module. `recordInjectionEvents` must never be called by
+// the simulate path — record any call so the assertion catches a regression.
+// `computeInjectionScores` is called twice per simulate (once inside runRouter
+// for tier 2, once in the handler for the response payload); always returns
+// zero scores since the test workspace has no event history.
+const recordCalls: Array<{ slugs: readonly string[]; at: number }> = [];
+mock.module("../../../memory/v2/injection-events.js", () => ({
+  recordInjectionEvents: (
+    _db: unknown,
+    slugs: readonly string[],
+    at: number,
+  ) => {
+    recordCalls.push({ slugs, at });
+  },
+  computeInjectionScores: (
+    _db: unknown,
+    slugs: readonly string[],
+    _now: number,
+  ): Map<string, number> => new Map(slugs.map((s) => [s, 0])),
+}));
+
+// Database handle: the simulate route only passes this through to
+// `runRouter` and `computeInjectionScores`. Both are stubbed above, so a
+// sentinel object is sufficient.
+mock.module("../../../memory/db-connection.js", () => ({
+  getDb: () => ({ __stub: true }),
+}));
+
+// Config loader. The simulate route reads `memory.v2.enabled` (must be
+// true) and the full `memory.v2.router` block (overrides merged on top).
+const liveRouterConfig = {
+  enabled: true,
+  max_page_ids: 25,
+  router_prompt_path: null,
+  batch_size: null,
+  tier1_size: null,
+  tier2_size: null,
+};
+const mockConfigValue = {
+  memory: {
+    v2: {
+      enabled: true,
+      router: liveRouterConfig,
+    },
+  },
+};
+mock.module("../../../config/loader.js", () => ({
+  loadConfig: () => mockConfigValue,
+  getConfig: () => mockConfigValue,
+  getConfigReadOnly: () => mockConfigValue,
+  invalidateConfigCache: () => {},
+  API_KEY_PROVIDERS: [],
+}));
+
+// Provider stub. Default returns [1, 2] (selects first two pages).
+let providerStub: Provider | null = null;
+interface ProviderCall {
+  messages: Message[];
+  tools: ToolDefinition[] | undefined;
+  systemPrompt: string | undefined;
+  options: SendMessageOptions | undefined;
+}
+const providerCalls: ProviderCall[] = [];
+mock.module("../../../providers/provider-send-message.js", () => ({
+  getConfiguredProvider: async () => providerStub,
+  extractToolUse: (response: ProviderResponse) =>
+    response.content.find((b): b is ToolUseContent => b.type === "tool_use"),
+}));
+
+// Platform helpers. `getWorkspaceDir` must return the per-test tmp dir so
+// the route's page index points at the test workspace.
+let workspaceDir = "";
+mock.module("../../../util/platform.js", () => ({
+  getWorkspaceDir: () => workspaceDir,
+}));
+
+// ---------------------------------------------------------------------------
+// Import under test (after all mocks above)
+// ---------------------------------------------------------------------------
+
+const { handleSimulateRouter } = await import("../memory-v2-routes.js");
+const { writePage } = await import("../../../memory/v2/page-store.js");
+const { invalidatePageIndex } =
+  await import("../../../memory/v2/page-index.js");
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeProvider(pageIds: number[]): Provider {
+  return {
+    name: "stub",
+    sendMessage: async (messages, tools, systemPrompt, options) => {
+      providerCalls.push({ messages, tools, systemPrompt, options });
+      return {
+        model: "stub-model",
+        stopReason: "tool_use",
+        usage: { inputTokens: 0, outputTokens: 0 },
+        content: [
+          {
+            type: "tool_use",
+            id: "tu-1",
+            name: "select_pages_to_inject",
+            input: { page_ids: pageIds },
+          },
+        ],
+      };
+    },
+  };
+}
+
+function makePage(slug: string, summary: string) {
+  return {
+    slug,
+    frontmatter: {
+      edges: [],
+      ref_files: [],
+      ref_urls: [],
+      summary,
+    },
+    body: "",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Per-test setup / teardown
+// ---------------------------------------------------------------------------
+
+beforeEach(() => {
+  workspaceDir = mkdtempSync(join(tmpdir(), "memory-v2-simulate-test-"));
+  recordCalls.length = 0;
+  providerCalls.length = 0;
+  providerStub = null;
+  // Reset live router config to defaults between tests.
+  liveRouterConfig.batch_size = null;
+  liveRouterConfig.tier1_size = null;
+  liveRouterConfig.tier2_size = null;
+  liveRouterConfig.max_page_ids = 25;
+  invalidatePageIndex();
+});
+
+afterEach(() => {
+  invalidatePageIndex();
+  rmSync(workspaceDir, { recursive: true, force: true });
+});
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("handleSimulateRouter", () => {
+  test("returns selectedSlugs + sourceBySlug populated for each pick", async () => {
+    await writePage(workspaceDir, makePage("alice", "A"));
+    await writePage(workspaceDir, makePage("bob", "B"));
+    await writePage(workspaceDir, makePage("carol", "C"));
+    providerStub = makeProvider([3, 1]);
+
+    const result = await handleSimulateRouter({
+      body: { query: "what's relevant?" },
+    });
+
+    expect(result.failureReason).toBeNull();
+    expect(result.selectedSlugs).toEqual(["carol", "alice"]);
+    expect(result.sourceBySlug["carol"]).toBe("tier3:0");
+    expect(result.sourceBySlug["alice"]).toBe("tier3:0");
+    expect(result.scores["carol"]).toBe(0);
+    expect(result.scores["alice"]).toBe(0);
+    expect(result.totalCandidatePages).toBe(3);
+  });
+
+  test("propagates overrides into effectiveConfig and reports them in overrides", async () => {
+    await writePage(workspaceDir, makePage("alice", "A"));
+    providerStub = makeProvider([1]);
+
+    const result = await handleSimulateRouter({
+      body: {
+        query: "test",
+        configOverrides: {
+          tier1_size: 50,
+          batch_size: 25,
+        },
+      },
+    });
+
+    expect(result.effectiveConfig.tier1_size).toBe(50);
+    expect(result.effectiveConfig.tier2_size).toBeNull(); // not overridden, inherits live
+    expect(result.effectiveConfig.batch_size).toBe(25);
+    expect(result.effectiveConfig.max_page_ids).toBe(25);
+    expect(result.overrides).toEqual({ tier1_size: 50, batch_size: 25 });
+  });
+
+  test("never writes to the EMA event log (recordInjectionEvents not called)", async () => {
+    await writePage(workspaceDir, makePage("alice", "A"));
+    await writePage(workspaceDir, makePage("bob", "B"));
+    providerStub = makeProvider([1, 2]);
+
+    await handleSimulateRouter({
+      body: { query: "should not record" },
+    });
+
+    expect(recordCalls).toEqual([]);
+  });
+
+  test("rejects an empty query at the schema layer", async () => {
+    await expect(
+      handleSimulateRouter({ body: { query: "" } }),
+    ).rejects.toThrow();
+  });
+
+  test("rejects negative tier size at the schema layer", async () => {
+    await expect(
+      handleSimulateRouter({
+        body: { query: "test", configOverrides: { tier1_size: -5 } },
+      }),
+    ).rejects.toThrow();
+  });
+});

package/src/runtime/routes/__tests__/sanity-routes.test.ts

@@ -0,0 +1,280 @@
+/**
+ * Unit tests for sanity-routes.ts.
+ *
+ * Drives the handler functions directly (bypassing the router) and mocks
+ * out secure-keys, the node:fs writes, and fetch so no real I/O occurs.
+ */
+
+import {
+  afterAll,
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  mock,
+  test,
+} from "bun:test";
+
+// ---------------------------------------------------------------------------
+// Module mocks — must appear before any imports of the module under test
+// ---------------------------------------------------------------------------
+
+let storedToken: string | undefined = undefined;
+
+mock.module("../../../security/secure-keys.js", () => ({
+  getSecureKeyAsync: async (_key: string) => storedToken,
+}));
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+const FAKE_WORKSPACE = "/tmp/sanity-routes-test-workspace";
+
+mock.module("../../../util/platform.js", () => ({
+  getWorkspaceDir: () => FAKE_WORKSPACE,
+}));
+
+const writtenFiles = new Map<string, string>();
+
+mock.module("node:fs", () => ({
+  mkdirSync: () => {},
+  writeFileSync: (path: string, content: string) => {
+    writtenFiles.set(path, content);
+  },
+  // Expose the rest of node:fs unchanged so other imports still work.
+  // Bun's mock.module replaces only what we return, and the module under
+  // test only imports mkdirSync and writeFileSync from node:fs.
+}));
+
+// ---------------------------------------------------------------------------
+// Imports after mocks
+// ---------------------------------------------------------------------------
+
+import { ROUTES } from "../sanity-routes.js";
+import type { RouteDefinition, RouteHandlerArgs } from "../types.js";
+
+afterAll(() => {
+  mock.restore();
+});
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function findHandler(operationId: string): RouteDefinition["handler"] {
+  const route = ROUTES.find((r) => r.operationId === operationId);
+  if (!route) throw new Error(`Route ${operationId} not found`);
+  return route.handler;
+}
+
+function makeArgs(body?: Record<string, unknown>): RouteHandlerArgs {
+  return { body };
+}
+
+const discoverHandler = findHandler("sanity_discover");
+const connectHandler = findHandler("sanity_connect");
+
+// ---------------------------------------------------------------------------
+// Fetch mock helpers
+// ---------------------------------------------------------------------------
+
+type FetchReturn = { status: number; json: () => unknown };
+let mockFetchImpl: ((url: string) => FetchReturn) | undefined = undefined;
+
+const originalFetch = globalThis.fetch;
+
+function mockFetch(url: string | URL | Request): Promise<Response> {
+  const urlStr =
+    typeof url === "string" ? url : url instanceof URL ? url.href : url.url;
+  if (!mockFetchImpl) throw new Error(`Unexpected fetch: ${urlStr}`);
+  const result = mockFetchImpl(urlStr);
+  return Promise.resolve({
+    ok: result.status >= 200 && result.status < 300,
+    status: result.status,
+    json: async () => result.json(),
+  } as Response);
+}
+
+beforeEach(() => {
+  writtenFiles.clear();
+  storedToken = undefined;
+  mockFetchImpl = undefined;
+
+  (globalThis as any).fetch = mockFetch;
+});
+
+afterEach(() => {
+  (globalThis as any).fetch = originalFetch;
+});
+
+// ---------------------------------------------------------------------------
+// POST /v1/sanity/discover — no stored token
+// ---------------------------------------------------------------------------
+
+describe("sanity_discover — no stored token", () => {
+  test("returns { error: 'no_token' } when token absent", async () => {
+    storedToken = undefined;
+    const result = await discoverHandler(makeArgs({}));
+    expect(result).toEqual({ error: "no_token" });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// POST /v1/sanity/discover — project listing
+// ---------------------------------------------------------------------------
+
+describe("sanity_discover — list projects", () => {
+  beforeEach(() => {
+    storedToken = "sk-test-token";
+  });
+
+  test("returns projects when API responds 200", async () => {
+    mockFetchImpl = () => ({
+      status: 200,
+      json: () => [
+        { id: "proj-a", displayName: "Project Alpha" },
+        { id: "proj-b", displayName: "Project Beta" },
+      ],
+    });
+
+    const result = await discoverHandler(makeArgs({}));
+    expect(result).toEqual({
+      projects: [
+        { id: "proj-a", displayName: "Project Alpha" },
+        { id: "proj-b", displayName: "Project Beta" },
+      ],
+    });
+  });
+
+  test("returns { error: 'token_scope_limited' } on 401", async () => {
+    mockFetchImpl = () => ({ status: 401, json: () => ({}) });
+    const result = await discoverHandler(makeArgs({}));
+    expect(result).toEqual({ error: "token_scope_limited" });
+  });
+
+  test("returns { error: 'token_scope_limited' } on 403", async () => {
+    mockFetchImpl = () => ({ status: 403, json: () => ({}) });
+    const result = await discoverHandler(makeArgs({}));
+    expect(result).toEqual({ error: "token_scope_limited" });
+  });
+
+  test("returns { error: 'discovery_failed' } on 500", async () => {
+    mockFetchImpl = () => ({ status: 500, json: () => ({}) });
+    const result = await discoverHandler(makeArgs({}));
+    expect(result).toEqual({ error: "discovery_failed" });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// POST /v1/sanity/discover — dataset listing (projectId provided)
+// ---------------------------------------------------------------------------
+
+describe("sanity_discover — list datasets", () => {
+  beforeEach(() => {
+    storedToken = "sk-test-token";
+  });
+
+  test("returns datasets when API responds 200", async () => {
+    mockFetchImpl = (url) => {
+      expect(url).toContain("/projects/my-project/datasets");
+      return {
+        status: 200,
+        json: () => [{ name: "production" }, { name: "staging" }],
+      };
+    };
+
+    const result = await discoverHandler(makeArgs({ projectId: "my-project" }));
+    expect(result).toEqual({
+      projectId: "my-project",
+      datasets: ["production", "staging"],
+    });
+  });
+
+  test("returns { error: 'token_scope_limited' } on 403 for dataset list", async () => {
+    mockFetchImpl = () => ({ status: 403, json: () => ({}) });
+    const result = await discoverHandler(makeArgs({ projectId: "my-project" }));
+    expect(result).toEqual({ error: "token_scope_limited" });
+  });
+
+  test("returns { error: 'discovery_failed' } on 500 for dataset list", async () => {
+    mockFetchImpl = () => ({ status: 500, json: () => ({}) });
+    const result = await discoverHandler(makeArgs({ projectId: "my-project" }));
+    expect(result).toEqual({ error: "discovery_failed" });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// POST /v1/sanity/connect
+// ---------------------------------------------------------------------------
+
+describe("sanity_connect", () => {
+  test("writes sidecar when token present and inputs valid", async () => {
+    storedToken = "sk-test-token";
+
+    const result = await connectHandler(
+      makeArgs({ projectId: "my-proj", dataset: "production" }),
+    );
+
+    expect(result).toEqual({ ok: true });
+
+    const expectedPath = `${FAKE_WORKSPACE}/data/sanity-connection.json`;
+    expect(writtenFiles.has(expectedPath)).toBe(true);
+    const written = JSON.parse(writtenFiles.get(expectedPath)!);
+    expect(written).toEqual({ projectId: "my-proj", dataset: "production" });
+  });
+
+  test("throws when no token is stored", async () => {
+    storedToken = undefined;
+
+    expect(
+      connectHandler(makeArgs({ projectId: "p", dataset: "production" })),
+    ).rejects.toThrow();
+  });
+
+  test("throws when projectId is empty", async () => {
+    storedToken = "sk-test-token";
+
+    expect(
+      connectHandler(makeArgs({ projectId: "  ", dataset: "production" })),
+    ).rejects.toThrow();
+  });
+
+  test("throws when dataset is empty", async () => {
+    storedToken = "sk-test-token";
+
+    expect(
+      connectHandler(makeArgs({ projectId: "my-proj", dataset: "" })),
+    ).rejects.toThrow();
+  });
+
+  test("does not return token in response", async () => {
+    storedToken = "sk-super-secret";
+
+    const result = (await connectHandler(
+      makeArgs({ projectId: "p", dataset: "d" }),
+    )) as Record<string, unknown>;
+
+    const resultStr = JSON.stringify(result);
+    expect(resultStr).not.toContain("sk-super-secret");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Policy key verification
+// ---------------------------------------------------------------------------
+
+describe("route policy keys", () => {
+  test("sanity_discover uses policyKey: secrets", () => {
+    const route = ROUTES.find((r) => r.operationId === "sanity_discover");
+    expect(route?.policyKey).toBe("secrets");
+  });
+
+  test("sanity_connect uses policyKey: secrets", () => {
+    const route = ROUTES.find((r) => r.operationId === "sanity_connect");
+    expect(route?.policyKey).toBe("secrets");
+  });
+});