@vellumai/assistant 0.8.3 → 0.8.5

package/src/memory/v3/__tests__/retriever.test.ts

@@ -0,0 +1,226 @@
+/**
+ * Route-assembly tests for the v3 retriever wiring in
+ * `handleCompareRetrievers` (`assistant/src/runtime/routes/memory-v2-routes.ts`).
+ *
+ * The compare route always includes the router retriever as comparand #1 and
+ * adds the v3 retriever as comparand #2 only when `config.memory.v3.enabled`.
+ * These tests exercise that gating end-to-end through the real handler and the
+ * real `runComparisonOverHistory`, with a fixture DB seeded with one logged
+ * router turn (mirroring `assistant/src/memory/v2/__tests__/harness-compare.test.ts`).
+ *
+ * Neither the real router nor the real v3 loop runs here — both would hit a
+ * provider. `../loop.js` (the v3 loop) and `../../v2/harness/router-retriever.js`
+ * are `mock.module`-stubbed to return deterministic selections, so the tests
+ * assert *which retrievers were assembled* (by the names in the report), not
+ * their retrieval quality. `loadConfig` is stubbed so each test controls
+ * `memory.v3.enabled`; workspace/page-index helpers are stubbed to keep the
+ * handler off the filesystem.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import type {
+  RetrievalInput,
+  RetrievalOutput,
+} from "../../v2/harness/retriever.js";
+
+initializeDb();
+
+// Silence the route's logger.
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+// loadNowText / page-index read workspace files; a nonexistent dir yields "".
+const WORKSPACE = "/tmp/v3-retriever-nonexistent-workspace";
+
+// Controllable config: each test sets `v3Enabled` before invoking the handler.
+let v3Enabled = false;
+
+mock.module("../../../config/loader.js", () => ({
+  loadConfig: (): AssistantConfig =>
+    ({
+      memory: {
+        v2: { enabled: true, router: { historical_pairs: 1 } },
+        v3: { enabled: v3Enabled },
+      },
+    }) as unknown as AssistantConfig,
+}));
+
+mock.module("../../../util/platform.js", () => ({
+  getWorkspaceDir: (): string => WORKSPACE,
+}));
+
+// page-index is intentionally NOT mocked: it has a wide export surface
+// (`invalidatePageIndex` etc.) that transitive importers in the route's
+// dependency graph rely on, and `getPageIndex` over the nonexistent workspace
+// returns a benign index. The retriever names are what we assert, not the
+// page set, so the real (empty-ish) index is harmless here.
+
+// Stub the router retriever — the real one calls a provider.
+mock.module("../../v2/harness/router-retriever.js", () => ({
+  createRouterRetriever: () => ({
+    name: "router",
+    retrieve: async (): Promise<RetrievalOutput> => ({
+      selectedSlugs: ["p1"],
+      sourceBySlug: new Map([["p1", "router"]]),
+    }),
+  }),
+}));
+
+// Stub the v3 loop — the real one runs scout/filter/tree/edge/gate lanes that
+// hit providers, embeddings, and the filesystem.
+mock.module("../loop.js", () => ({
+  runRetrievalLoop: async (
+    _input: RetrievalInput,
+  ): Promise<RetrievalOutput> => ({
+    selectedSlugs: ["p2"],
+    sourceBySlug: new Map([["p2", "dense"]]),
+  }),
+}));
+
+// Import the handler only after the mocks are installed.
+const { handleCompareRetrievers } =
+  await import("../../../runtime/routes/memory-v2-routes.js");
+
+const ZERO_CONFIG = {
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+};
+
+let seq = 0;
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  role: string,
+  text: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role,
+      content: JSON.stringify([{ type: "text", text }]),
+      createdAt,
+    })
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertRouterLog(
+  conversationId: string,
+  messageId: string,
+  turn: number,
+  concepts: MemoryV2ConceptRowRecord[],
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId,
+      messageId,
+      turn,
+      mode: "router",
+      conceptsJson: JSON.stringify(concepts),
+      skillsJson: "[]",
+      configJson: JSON.stringify(ZERO_CONFIG),
+      createdAt,
+    })
+    .run();
+}
+
+/** Seed one router turn: user msg, assistant anchor, and the logged picks. */
+function seedTurn(groundTruth: string[]): void {
+  insertMessage("u1", "c1", "user", "hello", 10);
+  insertMessage("a1", "c1", "assistant", "hi", 20); // anchor for turn 1
+  insertRouterLog(
+    "c1",
+    "a1",
+    1,
+    groundTruth.map((slug) => makeConcept(slug, "injected")),
+    20,
+  );
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+  v3Enabled = false;
+}
+
+describe("handleCompareRetrievers v3 wiring", () => {
+  beforeEach(reset);
+
+  test("includes only router when memory.v3.enabled is false", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await handleCompareRetrievers({ body: {} });
+
+    const names = report.retrievers.map((r) => r.name);
+    expect(names).toEqual(["router"]);
+  });
+
+  test("includes router and v3 when memory.v3.enabled is true", async () => {
+    v3Enabled = true;
+    seedTurn(["p1", "p2"]);
+
+    const report = await handleCompareRetrievers({ body: {} });
+
+    const names = report.retrievers.map((r) => r.name);
+    expect(names).toEqual(["router", "v3"]);
+    // Router is always comparand #1; v3 joins as comparand #2.
+    expect(names[0]).toBe("router");
+    expect(names[1]).toBe("v3");
+  });
+});

package/src/memory/v3/__tests__/scouts.test.ts

@@ -0,0 +1,440 @@
+/**
+ * Tests for `assistant/src/memory/v3/scouts.ts`.
+ *
+ * The scout lanes read the v2 substrate (page index, injection-event EMA,
+ * Qdrant hybrid query, BM25, dense embed + calibration). Every one of those is
+ * stubbed via `mock.module` so the suite needs no real Qdrant, embedding
+ * backend, or LLM — and the SQLite-backed EMA is replaced by a hand-fed score
+ * map, so the injected `db` is an opaque sentinel the lane never dereferences.
+ *
+ * Coverage:
+ *   - hot lane: ranks the EMA score map desc; hits are candidates but NOT
+ *     sticky (the query-aware gate may drop them).
+ *   - sparse lane: reads sparseScore, ranks desc, flags near-exact hits
+ *     sticky + tree-bypass.
+ *   - dense lane: per-subtree quota caps off-domain hits; MMR diversifies.
+ *   - lane toggles: each disabled lane is fully suppressed (no ScoutResult).
+ *   - empty query / empty corpus short-circuits.
+ *   - honors AbortSignal.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { PageIndex } from "../../v2/page-index.js";
+import type { ConceptPageQueryResult } from "../../v2/qdrant.js";
+
+// ---------------------------------------------------------------------------
+// Substrate stubs — installed before importing the module under test.
+// ---------------------------------------------------------------------------
+
+// Per-call programmable substrate state. Each test rewires these before
+// calling runScouts; the mock factories below close over the live refs.
+let injectionScores = new Map<string, number>();
+let pageSlugs: string[] = [];
+let hybridHits: ConceptPageQueryResult[] = [];
+let embedCalls = 0;
+// Records the text the sparse lane keyed its BM25 query off, so tests can
+// assert it is the user turn only (not the NOW context).
+let lastBm25QueryText: string | null = null;
+
+mock.module("../../v2/injection-events.js", () => ({
+  computeInjectionScores: () => injectionScores,
+}));
+
+mock.module("../../v2/page-index.js", () => ({
+  getPageIndex: async (): Promise<PageIndex> => ({
+    entries: pageSlugs.map((slug, i) => ({
+      id: i + 1,
+      slug,
+      summary: "",
+      edges: [],
+      modifiedAt: 0,
+    })),
+    bySlug: new Map(),
+    byId: new Map(),
+    rendered: "",
+  }),
+}));
+
+mock.module("../../v2/qdrant.js", () => ({
+  hybridQueryConceptPages: async (): Promise<ConceptPageQueryResult[]> =>
+    hybridHits,
+}));
+
+mock.module("../../v2/sparse-bm25.js", () => ({
+  // Non-empty indices so the sparse/dense lanes don't short-circuit on an
+  // "empty query embedding". The values are irrelevant — the stubbed Qdrant
+  // query ignores them and returns `hybridHits` directly.
+  generateBm25QueryEmbedding: (text: string) => {
+    lastBm25QueryText = text;
+    return text.trim().length > 0
+      ? { indices: [1], values: [1] }
+      : { indices: [], values: [] };
+  },
+}));
+
+mock.module("../../embedding-backend.js", () => ({
+  embedWithBackend: async () => {
+    embedCalls += 1;
+    return { provider: "local", model: "stub", vectors: [[0.1, 0.2, 0.3]] };
+  },
+}));
+
+mock.module("../../anisotropy.js", () => ({
+  applyCorrectionIfCalibrated: async (vec: number[]) => vec,
+}));
+
+const { runScouts } = await import("../scouts.js");
+import type { RetrievalInput } from "../../v2/harness/retriever.js";
+import type { ScoutDeps } from "../scouts.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const DB_SENTINEL = { __opaque: true } as unknown as ScoutDeps["db"];
+const DEPS: ScoutDeps = { db: DB_SENTINEL };
+
+type Lanes = { hot: boolean; sparse: boolean; dense: boolean };
+
+function makeInput(opts?: {
+  userMessage?: string;
+  nowText?: string;
+  lanes?: Partial<Lanes>;
+  denseQuota?: { activeDomain: number; offDomain: number };
+  hotLimit?: number;
+  signal?: AbortSignal;
+}): RetrievalInput {
+  const lanes = {
+    hot: true,
+    sparse: true,
+    dense: true,
+    tree: true,
+    edges: true,
+    ...opts?.lanes,
+  };
+  const config = {
+    memory: {
+      v3: {
+        lanes,
+        denseQuota: opts?.denseQuota ?? { activeDomain: 30, offDomain: 8 },
+        hotLimit: opts?.hotLimit ?? 50,
+      },
+    },
+  } as unknown as RetrievalInput["config"];
+  return {
+    workspaceDir: "/tmp/ws",
+    recentTurnPairs: [
+      { assistantMessage: "", userMessage: opts?.userMessage ?? "tell me" },
+    ],
+    nowText: opts?.nowText ?? "now context",
+    priorEverInjected: [],
+    config,
+    signal: opts?.signal,
+  };
+}
+
+function hit(
+  slug: string,
+  scores: Partial<ConceptPageQueryResult>,
+): ConceptPageQueryResult {
+  return { slug, ...scores };
+}
+
+beforeEach(() => {
+  injectionScores = new Map();
+  pageSlugs = [];
+  hybridHits = [];
+  embedCalls = 0;
+});
+
+// ---------------------------------------------------------------------------
+// Hot lane
+// ---------------------------------------------------------------------------
+
+describe("runScouts — hot lane", () => {
+  test("ranks EMA scores desc and seeds candidates without marking them sticky", async () => {
+    pageSlugs = ["people/alice", "work/proj", "essentials"];
+    injectionScores = new Map([
+      ["work/proj", 0.2],
+      ["people/alice", 0.9],
+    ]);
+
+    const { scouts, sticky } = await runScouts(
+      makeInput({ lanes: { sparse: false, dense: false } }),
+      DEPS,
+    );
+
+    const hot = scouts.find((s) => s.lane === "hot");
+    expect(hot?.slugs).toEqual(["people/alice", "work/proj"]);
+    expect(hot?.scoreBySlug).toEqual({ "people/alice": 0.9, "work/proj": 0.2 });
+    // Hot hits are candidates only — the query-aware gate may still drop them,
+    // so they must NOT be force-kept via sticky.
+    expect(sticky.size).toBe(0);
+  });
+
+  test("caps the hot lane to the top hotLimit by EMA", async () => {
+    pageSlugs = ["a", "b", "c", "d"];
+    injectionScores = new Map([
+      ["a", 0.9],
+      ["b", 0.7],
+      ["c", 0.5],
+      ["d", 0.3],
+    ]);
+
+    const { scouts, sticky } = await runScouts(
+      makeInput({ lanes: { sparse: false, dense: false }, hotLimit: 2 }),
+      DEPS,
+    );
+
+    // Only the top-2 by EMA survive the cap; the long tail is dropped so it
+    // can't flood the candidate set on a mature corpus.
+    const hot = scouts.find((s) => s.lane === "hot");
+    expect(hot?.slugs).toEqual(["a", "b"]);
+    expect(hot?.scoreBySlug).toEqual({ a: 0.9, b: 0.7 });
+    // Even the capped hot hits are not sticky.
+    expect(sticky.size).toBe(0);
+  });
+
+  test("empty corpus yields no hot ScoutResult", async () => {
+    pageSlugs = [];
+    const { scouts } = await runScouts(
+      makeInput({ lanes: { sparse: false, dense: false } }),
+      DEPS,
+    );
+    expect(scouts.find((s) => s.lane === "hot")).toBeUndefined();
+  });
+
+  test("no EMA events yields no hot ScoutResult", async () => {
+    pageSlugs = ["a", "b"];
+    injectionScores = new Map();
+    const { scouts, sticky } = await runScouts(
+      makeInput({ lanes: { sparse: false, dense: false } }),
+      DEPS,
+    );
+    expect(scouts.find((s) => s.lane === "hot")).toBeUndefined();
+    expect(sticky.size).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Sparse lane
+// ---------------------------------------------------------------------------
+
+describe("runScouts — sparse lane", () => {
+  test("reads sparseScore, ranks desc, flags near-exact sticky + bypass", async () => {
+    hybridHits = [
+      hit("docs/readme", { sparseScore: 4.0 }),
+      hit("docs/api", { sparseScore: 3.9 }), // within 90% of top -> near-exact
+      hit("misc/note", { sparseScore: 1.0 }), // below threshold
+      hit("dense/only", { denseScore: 0.8 }), // no sparseScore -> dropped
+    ];
+
+    const { scouts, sticky, bypass } = await runScouts(
+      makeInput({ lanes: { hot: false, dense: false } }),
+      DEPS,
+    );
+
+    const sparse = scouts.find((s) => s.lane === "sparse");
+    expect(sparse?.slugs).toEqual(["docs/readme", "docs/api", "misc/note"]);
+    // Near-exact: readme (top) and api (>= 90% of top). Not misc/note.
+    expect([...sticky].sort()).toEqual(["docs/api", "docs/readme"]);
+    expect([...bypass].sort()).toEqual(["docs/api", "docs/readme"]);
+  });
+
+  test("keys the BM25 query off the user turn only, not the NOW context", async () => {
+    hybridHits = [hit("docs/readme", { sparseScore: 4.0 })];
+
+    await runScouts(
+      makeInput({
+        userMessage: "favorite foods",
+        nowText: "ongoing project alpha and journal beta",
+        lanes: { hot: false, dense: false },
+      }),
+      DEPS,
+    );
+
+    // The sparse lane must search the user's words alone — folding NOW in would
+    // make NOW-referenced pages near-exact (sticky) on every turn.
+    expect(lastBm25QueryText).toBe("favorite foods");
+  });
+
+  test("no sparse hits yields no sparse ScoutResult", async () => {
+    hybridHits = [hit("dense/only", { denseScore: 0.5 })];
+    const { scouts, sticky, bypass } = await runScouts(
+      makeInput({ lanes: { hot: false, dense: false } }),
+      DEPS,
+    );
+    expect(scouts.find((s) => s.lane === "sparse")).toBeUndefined();
+    expect(sticky.size).toBe(0);
+    expect(bypass.size).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Dense lane
+// ---------------------------------------------------------------------------
+
+describe("runScouts — dense lane", () => {
+  test("embeds the query and emits dense hits ranked by denseScore", async () => {
+    hybridHits = [
+      hit("work/a", { denseScore: 0.9 }),
+      hit("work/b", { denseScore: 0.7 }),
+    ];
+    const { scouts } = await runScouts(
+      makeInput({ lanes: { hot: false, sparse: false } }),
+      DEPS,
+    );
+    expect(embedCalls).toBe(1);
+    const dense = scouts.find((s) => s.lane === "dense");
+    expect(dense?.slugs[0]).toBe("work/a");
+    expect(dense?.scoreBySlug).toEqual({ "work/a": 0.9, "work/b": 0.7 });
+  });
+
+  test("per-subtree quota caps off-domain hits", async () => {
+    // Active domain = top hit's domain = "work". Off-domain quota = 1.
+    hybridHits = [
+      hit("work/a", { denseScore: 0.99 }),
+      hit("work/b", { denseScore: 0.98 }),
+      hit("work/c", { denseScore: 0.97 }),
+      hit("people/x", { denseScore: 0.5 }), // off-domain, claims the 1 slot
+      hit("notes/y", { denseScore: 0.4 }), // off-domain, over quota -> dropped
+      hit("misc/z", { denseScore: 0.3 }), // off-domain, over quota -> dropped
+    ];
+    const { scouts } = await runScouts(
+      makeInput({
+        lanes: { hot: false, sparse: false },
+        denseQuota: { activeDomain: 30, offDomain: 1 },
+      }),
+      DEPS,
+    );
+    const dense = scouts.find((s) => s.lane === "dense");
+    const slugs = dense?.slugs ?? [];
+    // All three work/* survive (active quota 30); exactly one off-domain hit.
+    expect(slugs.filter((s) => s.startsWith("work/")).length).toBe(3);
+    const offDomain = slugs.filter((s) => !s.startsWith("work/"));
+    expect(offDomain).toEqual(["people/x"]);
+  });
+
+  test("active-domain quota caps same-subtree hits", async () => {
+    hybridHits = [
+      hit("work/a", { denseScore: 0.99 }),
+      hit("work/b", { denseScore: 0.98 }),
+      hit("work/c", { denseScore: 0.97 }), // over active quota 2 -> dropped
+      hit("people/x", { denseScore: 0.5 }),
+    ];
+    const { scouts } = await runScouts(
+      makeInput({
+        lanes: { hot: false, sparse: false },
+        denseQuota: { activeDomain: 2, offDomain: 8 },
+      }),
+      DEPS,
+    );
+    const slugs = scouts.find((s) => s.lane === "dense")?.slugs ?? [];
+    expect(slugs.filter((s) => s.startsWith("work/")).length).toBe(2);
+    expect(slugs).toContain("people/x");
+  });
+
+  test("MMR interleaves subtrees rather than emitting a same-subtree run", async () => {
+    // Five work/* then one people/* of comparable relevance. Pure score order
+    // would bury people/x last; MMR should pull it forward once work/ is
+    // over-represented.
+    hybridHits = [
+      hit("work/a", { denseScore: 0.95 }),
+      hit("work/b", { denseScore: 0.94 }),
+      hit("work/c", { denseScore: 0.93 }),
+      hit("work/d", { denseScore: 0.92 }),
+      hit("people/x", { denseScore: 0.9 }),
+    ];
+    const { scouts } = await runScouts(
+      makeInput({
+        lanes: { hot: false, sparse: false },
+        denseQuota: { activeDomain: 30, offDomain: 8 },
+      }),
+      DEPS,
+    );
+    const slugs = scouts.find((s) => s.lane === "dense")?.slugs ?? [];
+    // people/x is not stranded at the very end despite the lowest raw score.
+    expect(slugs.indexOf("people/x")).toBeLessThan(slugs.length - 1);
+  });
+
+  test("no dense hits yields no dense ScoutResult", async () => {
+    hybridHits = [hit("sparse/only", { sparseScore: 2.0 })];
+    const { scouts } = await runScouts(
+      makeInput({ lanes: { hot: false, sparse: false } }),
+      DEPS,
+    );
+    expect(scouts.find((s) => s.lane === "dense")).toBeUndefined();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Lane toggles
+// ---------------------------------------------------------------------------
+
+describe("runScouts — lane toggles", () => {
+  test("disabling a lane suppresses its ScoutResult", async () => {
+    pageSlugs = ["a"];
+    injectionScores = new Map([["a", 1]]);
+    hybridHits = [hit("docs/a", { sparseScore: 2.0, denseScore: 0.5 })];
+
+    const all = await runScouts(makeInput(), DEPS);
+    expect(all.scouts.map((s) => s.lane).sort()).toEqual([
+      "dense",
+      "hot",
+      "sparse",
+    ]);
+
+    const hotOnly = await runScouts(
+      makeInput({ lanes: { sparse: false, dense: false } }),
+      DEPS,
+    );
+    expect(hotOnly.scouts.map((s) => s.lane)).toEqual(["hot"]);
+    // Dense embed must not run when the dense lane is off.
+    embedCalls = 0;
+    await runScouts(makeInput({ lanes: { dense: false } }), DEPS);
+    expect(embedCalls).toBe(0);
+  });
+
+  test("all lanes off yields empty result", async () => {
+    pageSlugs = ["a"];
+    injectionScores = new Map([["a", 1]]);
+    hybridHits = [hit("docs/a", { sparseScore: 2.0, denseScore: 0.5 })];
+    const { scouts, sticky, bypass } = await runScouts(
+      makeInput({ lanes: { hot: false, sparse: false, dense: false } }),
+      DEPS,
+    );
+    expect(scouts).toEqual([]);
+    expect(sticky.size).toBe(0);
+    expect(bypass.size).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Misc
+// ---------------------------------------------------------------------------
+
+describe("runScouts — misc", () => {
+  test("empty query text skips sparse and dense lanes", async () => {
+    pageSlugs = ["a"];
+    injectionScores = new Map([["a", 1]]);
+    hybridHits = [hit("docs/a", { sparseScore: 2.0, denseScore: 0.5 })];
+    const { scouts } = await runScouts(
+      makeInput({ userMessage: "   ", nowText: "  " }),
+      DEPS,
+    );
+    // Hot lane is query-independent and still fires; sparse/dense are gated off.
+    expect(scouts.map((s) => s.lane)).toEqual(["hot"]);
+    expect(embedCalls).toBe(0);
+  });
+
+  test("honors an already-aborted signal", async () => {
+    const controller = new AbortController();
+    controller.abort();
+    pageSlugs = ["a"];
+    injectionScores = new Map([["a", 1]]);
+    await expect(
+      runScouts(makeInput({ signal: controller.signal }), DEPS),
+    ).rejects.toThrow();
+  });
+});