@vellumai/assistant 0.8.9-staging.2 → 0.8.9-staging.4

package/src/plugins/defaults/memory-v3-shadow/__tests__/selector.test.ts

@@ -1,470 +0,0 @@
-/**
- * Tests for `assistant/src/memory/v3/selector.ts`.
- *
- * Coverage matrix:
- *   - Returned IDs map to the right member slugs by 1-based index, with
- *     `pinned` driven by `pinned_ids`.
- *   - Omitted `ids` → ALL members of the leaf (recall-safe; bounded to one
- *     leaf, so this stays a select-all unlike the L1 router).
- *   - Explicit `ids: []` → no pages (deliberate abstention).
- *   - No provider / missing tool_use / schema mismatch / throw → no pages
- *     (degrade to the deterministic lanes), the last three after a re-prompt
- *     retry; a malformed response that recovers on retry returns its pages.
- *   - The per-leaf `<pages>` prefix is byte-identical across two calls with
- *     different turns (the cache invariant).
- *   - `selectAcrossLeaves` flattens per-leaf results and never exceeds the
- *     configured concurrency.
- *
- * The provider is stubbed so no network calls fire.
- */
-
-import { beforeEach, describe, expect, mock, test } from "bun:test";
-
-import type {
-  Message,
-  Provider,
-  ProviderResponse,
-  SendMessageOptions,
-  ToolUseContent,
-} from "../../../../providers/types.js";
-import type {
-  LeafNode,
-  LeafPath,
-  LeafTree,
-  MemoryRoutingTurn,
-  Slug,
-} from "../types.js";
-
-// ---------------------------------------------------------------------------
-// Mocks installed BEFORE the selector import so the module observes them at
-// load time.
-// ---------------------------------------------------------------------------
-
-let providerStub: Provider | null = null;
-
-interface ProviderCall {
-  messages: Message[];
-  options: SendMessageOptions | undefined;
-}
-const providerCalls: ProviderCall[] = [];
-
-mock.module("../../../../providers/provider-send-message.js", () => ({
-  getConfiguredProvider: async () => providerStub,
-  extractToolUse: (response: ProviderResponse) =>
-    response.content.find((b): b is ToolUseContent => b.type === "tool_use"),
-}));
-
-mock.module("../../../../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: (_t, prop) => (prop === "child" ? () => ({}) : () => {}),
-    }),
-}));
-
-const { selectFromLeaf, selectAcrossLeaves } = await import("../selector.js");
-
-// ---------------------------------------------------------------------------
-// Helpers.
-// ---------------------------------------------------------------------------
-
-function makeProvider(response: ProviderResponse): Provider {
-  return {
-    name: "stub",
-    sendMessage: async (messages, options) => {
-      providerCalls.push({ messages, options });
-      return response;
-    },
-  };
-}
-
-function toolUseResponse(input: Record<string, unknown>): ProviderResponse {
-  return {
-    model: "stub-model",
-    stopReason: "tool_use",
-    usage: { inputTokens: 0, outputTokens: 0 },
-    content: [{ type: "tool_use", id: "tu-1", name: "select_pages", input }],
-  };
-}
-
-/** A 200 response that carries no tool_use — the malformed-but-successful case
- * the re-prompt retry exists to recover from. */
-function noToolResponse(): ProviderResponse {
-  return {
-    model: "stub-model",
-    stopReason: "end_turn",
-    usage: { inputTokens: 0, outputTokens: 0 },
-    content: [{ type: "text", text: "no tool call" }],
-  };
-}
-
-/** Provider returning a different response per call (the i-th call returns
- * responses[i], or the last entry once exhausted), recording each call so a
- * test can assert how many attempts were made. */
-function makeSequenceProvider(responses: ProviderResponse[]): Provider {
-  let i = 0;
-  return {
-    name: "sequence",
-    sendMessage: async (messages, options) => {
-      providerCalls.push({ messages, options });
-      const response = responses[Math.min(i, responses.length - 1)];
-      i += 1;
-      return response;
-    },
-  };
-}
-
-/** Provider that records each call and then throws — for the throw-after-retries
- * path (the provider's own RetryProvider has already exhausted its backoff). */
-function makeThrowingProvider(): Provider {
-  return {
-    name: "throwing",
-    sendMessage: async (messages, options) => {
-      providerCalls.push({ messages, options });
-      throw new Error("boom");
-    },
-  };
-}
-
-function makeLeaf(path: LeafPath, members: Slug[]): LeafNode {
-  return {
-    path,
-    frontmatter: { path, in_core: false },
-    description: `description for ${path}`,
-    members,
-    domain: path.split("/")[0],
-  };
-}
-
-/** Tree with two leaves; `people/alice` has three member pages. */
-function makeTree(): LeafTree {
-  const leaves: LeafNode[] = [
-    makeLeaf("people/alice", ["alice-bio", "alice-1on1", "alice-feedback"]),
-    makeLeaf("projects/atlas", ["atlas-roadmap", "atlas-status"]),
-  ];
-  const byPage = new Map<Slug, LeafPath[]>();
-  for (const leaf of leaves) {
-    for (const slug of leaf.members) {
-      byPage.set(slug, [...(byPage.get(slug) ?? []), leaf.path]);
-    }
-  }
-  return {
-    leaves: new Map(leaves.map((n) => [n.path, n])),
-    byPage,
-  };
-}
-
-const ALICE_MEMBERS = ["alice-bio", "alice-1on1", "alice-feedback"];
-
-function makeTurn(currentMessage: string): MemoryRoutingTurn {
-  return {
-    conversationId: "conv-xyz",
-    turnNumber: 1,
-    currentMessage,
-    recentContext: "earlier we talked about the timeline",
-  };
-}
-
-/** Deterministic summary stub so the rendered prefix is reproducible. */
-const summaryOf = async (slug: Slug): Promise<string> => `summary of ${slug}`;
-
-beforeEach(() => {
-  providerStub = null;
-  providerCalls.length = 0;
-});
-
-// ---------------------------------------------------------------------------
-// selectFromLeaf — id mapping.
-// ---------------------------------------------------------------------------
-
-describe("selectFromLeaf — id mapping", () => {
-  test("returned IDs map to member slugs, pinned driven by pinned_ids", async () => {
-    providerStub = makeProvider(
-      toolUseResponse({ ids: [3, 1], pinned_ids: [1] }),
-    );
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("how's alice?"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([
-      { slug: "alice-feedback", pinned: false },
-      { slug: "alice-bio", pinned: true },
-    ]);
-  });
-
-  test("omitted ids selects ALL members (recall-safe)", async () => {
-    providerStub = makeProvider(toolUseResponse({}));
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("anything"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual(
-      ALICE_MEMBERS.map((slug) => ({ slug, pinned: false })),
-    );
-  });
-
-  test("explicit empty ids selects no pages (abstention)", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [] }));
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("nothing relevant"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([]);
-  });
-
-  test("out-of-range and duplicate IDs are ignored without throwing", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [2, 99, 0, -1, 2] }));
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("the 1:1"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([{ slug: "alice-1on1", pinned: false }]);
-  });
-
-  test("empty leaf returns no pages and never calls the provider", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [1] }));
-    const result = await selectFromLeaf(
-      "people/empty",
-      makeTurn("hi"),
-      {
-        leaves: new Map([["people/empty", makeLeaf("people/empty", [])]]),
-        byPage: new Map(),
-      },
-      summaryOf,
-    );
-    expect(result).toEqual([]);
-    expect(providerCalls).toHaveLength(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// selectFromLeaf — recall-safe fallbacks.
-// ---------------------------------------------------------------------------
-
-describe("selectFromLeaf — degradation on failure", () => {
-  test("no provider → no pages, without calling the provider", async () => {
-    providerStub = null;
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("x"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([]);
-    expect(providerCalls).toHaveLength(0);
-  });
-
-  test("missing tool_use → no pages after retrying", async () => {
-    providerStub = makeProvider(noToolResponse());
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("x"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([]);
-    expect(providerCalls).toHaveLength(3);
-  });
-
-  test("schema mismatch → no pages after retrying", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: "not-an-array" }));
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("x"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([]);
-    expect(providerCalls).toHaveLength(3);
-  });
-
-  test("provider throw → no pages after retrying", async () => {
-    providerStub = makeThrowingProvider();
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("x"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([]);
-    expect(providerCalls).toHaveLength(3);
-  });
-
-  test("a malformed response that recovers on retry returns its pages", async () => {
-    providerStub = makeSequenceProvider([
-      noToolResponse(),
-      toolUseResponse({ ids: [2] }),
-    ]);
-    const result = await selectFromLeaf(
-      "people/alice",
-      makeTurn("the 1:1"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result).toEqual([{ slug: "alice-1on1", pinned: false }]);
-    expect(providerCalls).toHaveLength(2);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// selectFromLeaf — request shape and cache invariant.
-// ---------------------------------------------------------------------------
-
-describe("selectFromLeaf — request shape", () => {
-  test("forces tool_choice to select_pages with the v3 L2 call site", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [1] }));
-    await selectFromLeaf(
-      "people/alice",
-      makeTurn("alice?"),
-      makeTree(),
-      summaryOf,
-    );
-
-    expect(providerCalls).toHaveLength(1);
-    const [call] = providerCalls;
-    const cfg = call.options?.config as Record<string, unknown>;
-    expect(cfg?.callSite).toBe("memoryV3SelectL2");
-    expect(cfg?.tool_choice).toEqual({ type: "tool", name: "select_pages" });
-    expect(call.options?.tools?.[0]?.name).toBe("select_pages");
-  });
-
-  test("pages block is the first content block with an ephemeral cache breakpoint", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [1] }));
-    await selectFromLeaf(
-      "people/alice",
-      makeTurn("alice?"),
-      makeTree(),
-      summaryOf,
-    );
-
-    const [blockA, blockB] = providerCalls[0].messages[0].content as Array<{
-      type: string;
-      text: string;
-      cache_control?: { type: string; ttl?: string };
-    }>;
-    expect(blockA.type).toBe("text");
-    expect(blockA.text).toContain("<leaf>people/alice</leaf>");
-    expect(blockA.text).toContain("<pages>");
-    expect(blockA.text).toContain("[1] alice-bio — summary of alice-bio");
-    expect(blockA.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
-
-    expect(blockB.type).toBe("text");
-    expect(blockB.text).toContain("<current_message>alice?</current_message>");
-    expect(blockB.text).toContain("<recent_context>");
-    expect(blockB.cache_control).toBeUndefined();
-  });
-
-  test("situational context renders in the per-turn block when present", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [1] }));
-    await selectFromLeaf(
-      "people/alice",
-      {
-        ...makeTurn("alice?"),
-        situationalContext: "Today is Saturday. Alice's anniversary is today.",
-      },
-      makeTree(),
-      summaryOf,
-    );
-    const blockB = providerCalls[0].messages[0].content[1] as { text: string };
-    expect(blockB.text).toContain(
-      "<situation>Today is Saturday. Alice's anniversary is today.</situation>",
-    );
-  });
-
-  test("system prompt mentions pinned (locks the pinning commitment)", async () => {
-    providerStub = makeProvider(toolUseResponse({ ids: [1] }));
-    await selectFromLeaf("people/alice", makeTurn("x"), makeTree(), summaryOf);
-    expect(providerCalls[0].options?.systemPrompt).toMatch(/pinned/);
-  });
-});
-
-describe("selectFromLeaf — per-leaf cache invariant", () => {
-  test("the pages prefix is byte-identical across two calls with different turns", async () => {
-    const tree = makeTree();
-    providerStub = makeProvider(toolUseResponse({ ids: [1] }));
-
-    await selectFromLeaf(
-      "people/alice",
-      makeTurn("first query"),
-      tree,
-      summaryOf,
-    );
-    await selectFromLeaf(
-      "people/alice",
-      makeTurn("totally different second query"),
-      tree,
-      summaryOf,
-    );
-
-    const prefixA = (
-      providerCalls[0].messages[0].content[0] as { text: string }
-    ).text;
-    const prefixB = (
-      providerCalls[1].messages[0].content[0] as { text: string }
-    ).text;
-    expect(prefixA).toBe(prefixB);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// selectAcrossLeaves — fan-out and bounded concurrency.
-// ---------------------------------------------------------------------------
-
-describe("selectAcrossLeaves", () => {
-  test("flattens per-leaf selections", async () => {
-    // The selector forces the tool, so omitted ids → all members per leaf.
-    providerStub = makeProvider(toolUseResponse({}));
-    const result = await selectAcrossLeaves(
-      ["people/alice", "projects/atlas"],
-      makeTurn("x"),
-      makeTree(),
-      summaryOf,
-    );
-    expect(result.map((p) => p.slug)).toEqual([
-      "alice-bio",
-      "alice-1on1",
-      "alice-feedback",
-      "atlas-roadmap",
-      "atlas-status",
-    ]);
-  });
-
-  test("never exceeds the configured concurrency", async () => {
-    let inFlight = 0;
-    let maxInFlight = 0;
-    let calls = 0;
-    providerStub = {
-      name: "tracking",
-      sendMessage: async () => {
-        calls++;
-        inFlight++;
-        maxInFlight = Math.max(maxInFlight, inFlight);
-        await new Promise((r) => setTimeout(r, 5));
-        inFlight--;
-        return toolUseResponse({ ids: [] });
-      },
-    };
-
-    // Build a tree with eight single-member leaves so there are eight calls.
-    const leaves: LeafNode[] = Array.from({ length: 8 }, (_, i) =>
-      makeLeaf(`misc/leaf-${i}`, [`page-${i}`]),
-    );
-    const tree: LeafTree = {
-      leaves: new Map(leaves.map((n) => [n.path, n])),
-      byPage: new Map(),
-    };
-    const paths = leaves.map((l) => l.path);
-
-    await selectAcrossLeaves(paths, makeTurn("x"), tree, summaryOf, 3);
-    expect(maxInFlight).toBeLessThanOrEqual(3);
-    expect(maxInFlight).toBeGreaterThan(1);
-    expect(calls).toBe(8);
-  });
-});

package/src/plugins/defaults/memory-v3-shadow/__tests__/snapshot.test.ts

@@ -1,168 +0,0 @@
-import { promises as fs } from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-
-import { restoreDataDir, snapshotDataDir } from "../snapshot.js";
-
-let tmpRoot: string;
-let dataDir: string;
-
-async function readJson(file: string): Promise<unknown> {
-  return JSON.parse(await fs.readFile(file, "utf8")) as unknown;
-}
-
-async function listLeaves(dir: string): Promise<Record<string, string>> {
-  const leavesDir = path.join(dir, "leaves");
-  const out: Record<string, string> = {};
-  async function walk(current: string, prefix: string): Promise<void> {
-    let entries: import("node:fs").Dirent[];
-    try {
-      entries = await fs.readdir(current, { withFileTypes: true });
-    } catch {
-      return;
-    }
-    for (const entry of entries) {
-      const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
-      const abs = path.join(current, entry.name);
-      if (entry.isDirectory()) {
-        await walk(abs, rel);
-      } else if (entry.isFile()) {
-        out[rel] = await fs.readFile(abs, "utf8");
-      }
-    }
-  }
-  await walk(leavesDir, "");
-  return out;
-}
-
-async function seedDataDir(): Promise<void> {
-  await fs.mkdir(path.join(dataDir, "leaves", "alpha"), { recursive: true });
-  await fs.writeFile(
-    path.join(dataDir, "leaves", "alpha", "leaf-1.md"),
-    "leaf one body",
-  );
-  await fs.writeFile(
-    path.join(dataDir, "leaves", "leaf-2.md"),
-    "leaf two body",
-  );
-  await fs.writeFile(
-    path.join(dataDir, "assignments.json"),
-    JSON.stringify({ "leaf-1": ["topic-a"] }),
-  );
-  await fs.writeFile(
-    path.join(dataDir, "core.json"),
-    JSON.stringify({ version: 1, root: "topic-a" }),
-  );
-}
-
-beforeEach(async () => {
-  tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "v3-snapshot-"));
-  dataDir = path.join(tmpRoot, "data");
-  await fs.mkdir(dataDir, { recursive: true });
-});
-
-afterEach(async () => {
-  await fs.rm(tmpRoot, { recursive: true, force: true });
-});
-
-describe("snapshotDataDir / restoreDataDir", () => {
-  test("roundtrips leaves, assignments, and core", async () => {
-    await seedDataDir();
-    const before = {
-      leaves: await listLeaves(dataDir),
-      assignments: await readJson(path.join(dataDir, "assignments.json")),
-      core: await readJson(path.join(dataDir, "core.json")),
-    };
-
-    const snapshotPath = await snapshotDataDir(dataDir, { label: "snap-1" });
-
-    // Mutate the live data dir after snapshotting.
-    await fs.writeFile(
-      path.join(dataDir, "leaves", "alpha", "leaf-1.md"),
-      "CORRUPTED",
-    );
-    await fs.rm(path.join(dataDir, "leaves", "leaf-2.md"));
-    await fs.writeFile(
-      path.join(dataDir, "assignments.json"),
-      JSON.stringify({ broken: true }),
-    );
-    await fs.writeFile(
-      path.join(dataDir, "core.json"),
-      JSON.stringify({ broken: true }),
-    );
-
-    await restoreDataDir(snapshotPath, dataDir);
-
-    expect(await listLeaves(dataDir)).toEqual(before.leaves);
-    expect(await readJson(path.join(dataDir, "assignments.json"))).toEqual(
-      before.assignments,
-    );
-    expect(await readJson(path.join(dataDir, "core.json"))).toEqual(
-      before.core,
-    );
-  });
-
-  test("roundtrips the pageRefs diff", async () => {
-    await seedDataDir();
-    const pageRefs = new Map<string, string[]>([
-      ["slug-one", ["leaf-1", "leaf-2"]],
-      ["slug-two", ["leaf-3"]],
-    ]);
-
-    const snapshotPath = await snapshotDataDir(dataDir, {
-      label: "snap-refs",
-      pageRefs,
-    });
-
-    const { pageRefs: restored } = await restoreDataDir(snapshotPath, dataDir);
-    expect(restored).toBeDefined();
-    expect(restored).toEqual(pageRefs);
-  });
-
-  test("returns no pageRefs when none were captured", async () => {
-    await seedDataDir();
-    const snapshotPath = await snapshotDataDir(dataDir, { label: "snap-none" });
-    const result = await restoreDataDir(snapshotPath, dataDir);
-    expect(result.pageRefs).toBeUndefined();
-  });
-
-  test("restoring removes leaves added after the snapshot", async () => {
-    await seedDataDir();
-    const snapshotPath = await snapshotDataDir(dataDir, { label: "snap-add" });
-
-    await fs.writeFile(
-      path.join(dataDir, "leaves", "leaf-extra.md"),
-      "added later",
-    );
-
-    await restoreDataDir(snapshotPath, dataDir);
-    const leaves = await listLeaves(dataDir);
-    expect(leaves["leaf-extra.md"]).toBeUndefined();
-    expect(leaves["leaf-2.md"]).toBe("leaf two body");
-  });
-
-  test("retention is bounded to the last N snapshots", async () => {
-    await seedDataDir();
-    // Zero-padded labels sort chronologically.
-    const labels = Array.from(
-      { length: 8 },
-      (_, i) => `snap-${String(i).padStart(2, "0")}`,
-    );
-    for (const label of labels) {
-      await snapshotDataDir(dataDir, { label });
-    }
-
-    const snapshotsRoot = path.join(tmpRoot, "v3-snapshots");
-    const remaining = (await fs.readdir(snapshotsRoot)).sort();
-    expect(remaining.length).toBe(5);
-    // Oldest three pruned, newest five kept.
-    expect(remaining).toEqual([
-      "snap-03",
-      "snap-04",
-      "snap-05",
-      "snap-06",
-      "snap-07",
-    ]);
-  });
-});