@vellumai/assistant 0.8.4 → 0.8.5

package/src/memory/v2/__tests__/harness-compare.test.ts

@@ -0,0 +1,186 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({ memory: { enabled: false } }),
+}));
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import { runComparisonOverHistory } from "../harness/compare.js";
+import type { RetrievalOutput, Retriever } from "../harness/retriever.js";
+
+initializeDb();
+
+// loadNowText reads workspace files; a nonexistent dir yields "".
+const WORKSPACE = "/tmp/harness-compare-nonexistent-workspace";
+
+const ZERO_CONFIG = {
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+};
+
+let seq = 0;
+
+function config(historicalPairs: number): AssistantConfig {
+  return {
+    memory: { v2: { router: { historical_pairs: historicalPairs } } },
+  } as unknown as AssistantConfig;
+}
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  role: string,
+  text: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role,
+      content: JSON.stringify([{ type: "text", text }]),
+      createdAt,
+    })
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertRouterLog(
+  conversationId: string,
+  messageId: string,
+  turn: number,
+  concepts: MemoryV2ConceptRowRecord[],
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId,
+      messageId,
+      turn,
+      mode: "router",
+      conceptsJson: JSON.stringify(concepts),
+      skillsJson: "[]",
+      configJson: JSON.stringify(ZERO_CONFIG),
+      createdAt,
+    })
+    .run();
+}
+
+function stubRetriever(name: string, selected: string[]): Retriever {
+  return {
+    name,
+    retrieve: async (): Promise<RetrievalOutput> => ({
+      selectedSlugs: selected,
+      sourceBySlug: new Map(selected.map((s): [string, string] => [s, name])),
+    }),
+  };
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+}
+
+/** Seed one router turn: user msg, assistant anchor, and the logged picks. */
+function seedTurn(groundTruth: string[]): void {
+  insertMessage("u1", "c1", "user", "hello", 10);
+  insertMessage("a1", "c1", "assistant", "hi", 20); // anchor for turn 1
+  insertRouterLog(
+    "c1",
+    "a1",
+    1,
+    groundTruth.map((slug) => makeConcept(slug, "injected")),
+    20,
+  );
+}
+
+describe("harness/compare runComparisonOverHistory", () => {
+  beforeEach(reset);
+
+  test("scores a stub retriever against the logged ground truth", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await runComparisonOverHistory({
+      db: getDb(),
+      workspaceDir: WORKSPACE,
+      config: config(1),
+      retrievers: [stubRetriever("stub", ["p1", "z"])],
+      ks: [5],
+    });
+
+    expect(report.turnsConsidered).toBe(1);
+    expect(report.turnsScored).toBe(1);
+    expect(report.turnsSkipped).toBe(0);
+    // hit p1, miss p2 → recall@5 = 1/2
+    expect(report.retrievers[0]?.aggregate.meanRecallAtK[5]).toBeCloseTo(0.5);
+  });
+
+  test("pageExists narrows the ground truth set", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await runComparisonOverHistory({
+      db: getDb(),
+      workspaceDir: WORKSPACE,
+      config: config(1),
+      retrievers: [stubRetriever("stub", ["p1", "z"])],
+      ks: [5],
+      pageExists: (slug) => slug === "p1", // p2 no longer exists
+    });
+
+    // ground truth is just {p1}; stub selected p1 → recall 1.0
+    expect(report.retrievers[0]?.aggregate.meanRecallAtK[5]).toBeCloseTo(1);
+  });
+});

package/src/memory/v2/__tests__/harness-metrics.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test";
+
+import { aggregate, evalTurn, recallAtK } from "../harness/metrics.js";
+import type { RetrievalOutput } from "../harness/retriever.js";
+
+function out(
+  selected: string[],
+  sourceBySlug: Record<string, string> = {},
+): RetrievalOutput {
+  return {
+    selectedSlugs: selected,
+    sourceBySlug: new Map(Object.entries(sourceBySlug)),
+  };
+}
+
+describe("harness/metrics recallAtK", () => {
+  test("fraction of ground truth recovered within k", () => {
+    const gt = new Set(["a", "b", "c", "d"]);
+    expect(recallAtK(["a", "b", "x"], gt, 10)).toBeCloseTo(0.5);
+    // only the top-2 selections count toward recall@2
+    expect(recallAtK(["a", "b", "c", "d"], gt, 2)).toBeCloseTo(0.5);
+    expect(recallAtK(["a", "b", "c", "d"], gt, 10)).toBeCloseTo(1);
+  });
+
+  test("empty ground truth is vacuously complete (recall 1)", () => {
+    expect(recallAtK([], new Set<string>(), 5)).toBe(1);
+  });
+});
+
+describe("harness/metrics evalTurn", () => {
+  test("hits / misses / extras and per-lane attribution", () => {
+    const e = evalTurn(
+      out(["a", "b", "z"], { a: "tree", b: "sparse", z: "dense" }),
+      ["a", "b", "c"],
+      [5],
+    );
+    expect(e.hits.sort()).toEqual(["a", "b"]);
+    expect(e.misses).toEqual(["c"]);
+    // selected but not in ground truth — reported as diff, not error
+    expect(e.extras).toEqual(["z"]);
+    expect(e.recallAtK[5]).toBeCloseTo(2 / 3);
+    expect(e.hitsByLane).toEqual({ tree: 1, sparse: 1 });
+  });
+
+  test("failureReason surfaces and recall is 0", () => {
+    const o: RetrievalOutput = {
+      selectedSlugs: [],
+      sourceBySlug: new Map(),
+      failureReason: "no_provider",
+    };
+    const e = evalTurn(o, ["a"], [5]);
+    expect(e.failureReason).toBe("no_provider");
+    expect(e.recallAtK[5]).toBe(0);
+  });
+});
+
+describe("harness/metrics aggregate", () => {
+  test("means recall@k and failure rate across turns", () => {
+    const ks = [5];
+    const t1 = evalTurn(out(["a"]), ["a"], ks); // recall 1
+    const t2 = evalTurn(out([]), ["a"], ks); // recall 0
+    const agg = aggregate([t1, t2], ks);
+    expect(agg.turns).toBe(2);
+    expect(agg.meanRecallAtK[5]).toBeCloseTo(0.5);
+    expect(agg.failureRate).toBe(0);
+  });
+
+  test("empty input is well-defined", () => {
+    const agg = aggregate([], [5]);
+    expect(agg.turns).toBe(0);
+    expect(agg.meanRecallAtK[5]).toBe(0);
+    expect(agg.failureRate).toBe(0);
+  });
+});

package/src/memory/v2/__tests__/harness-oracle.test.ts

@@ -0,0 +1,257 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({ memory: { enabled: false } }),
+}));
+
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import { extractOracleTurns } from "../harness/oracle.js";
+
+initializeDb();
+
+const CONFIG_JSON = JSON.stringify({
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+});
+
+let seq = 0;
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertLog(opts: {
+  conversationId: string;
+  messageId: string | null;
+  turn: number;
+  mode: string;
+  concepts: MemoryV2ConceptRowRecord[];
+  createdAt: number;
+}): void {
+  ensureConversation(opts.conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId: opts.conversationId,
+      messageId: opts.messageId,
+      turn: opts.turn,
+      mode: opts.mode,
+      conceptsJson: JSON.stringify(opts.concepts),
+      skillsJson: "[]",
+      configJson: CONFIG_JSON,
+      createdAt: opts.createdAt,
+    })
+    .run();
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role: "assistant",
+      content: JSON.stringify([{ type: "text", text: "reply" }]),
+      createdAt,
+    })
+    .run();
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+}
+
+describe("harness/oracle extractOracleTurns", () => {
+  beforeEach(reset);
+
+  test("returns only mode='router' rows with a resolvable messageId", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      concepts: [makeConcept("a", "injected")],
+      createdAt: 100,
+    });
+    // non-router row — ignored
+    insertMessage("m2", "c1", 200);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m2",
+      turn: 2,
+      mode: "per-turn",
+      concepts: [makeConcept("b", "injected")],
+      createdAt: 200,
+    });
+    // router row, null messageId — skipped (no anchor)
+    insertLog({
+      conversationId: "c1",
+      messageId: null,
+      turn: 3,
+      mode: "router",
+      concepts: [makeConcept("c", "injected")],
+      createdAt: 300,
+    });
+    // router row, messageId does not resolve — skipped
+    insertLog({
+      conversationId: "c1",
+      messageId: "ghost",
+      turn: 4,
+      mode: "router",
+      concepts: [makeConcept("d", "injected")],
+      createdAt: 400,
+    });
+
+    const turns = extractOracleTurns(getDb(), { limit: 50 });
+    expect(turns.length).toBe(1);
+    expect(turns[0]?.turn).toBe(1);
+    expect(turns[0]?.groundTruthSlugs).toEqual(["a"]);
+    expect(turns[0]?.anchorMessageId).toBe("m1");
+    expect(turns[0]?.anchorCreatedAt).toBe(100);
+  });
+
+  test("ground truth keeps injected/in_context, drops the rest", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [
+        makeConcept("inj", "injected"),
+        makeConcept("ctx", "in_context"),
+        makeConcept("noinj", "not_injected"),
+        makeConcept("miss", "page_missing"),
+        makeConcept("bad", "corrupt"),
+      ],
+    });
+    const turns = extractOracleTurns(getDb());
+    expect(turns[0]?.groundTruthSlugs.sort()).toEqual(["ctx", "inj"]);
+  });
+
+  test("includeNotInjected adds not_injected to ground truth", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [
+        makeConcept("inj", "injected"),
+        makeConcept("noinj", "not_injected"),
+      ],
+    });
+    const turns = extractOracleTurns(getDb(), { includeNotInjected: true });
+    expect(turns[0]?.groundTruthSlugs.sort()).toEqual(["inj", "noinj"]);
+  });
+
+  test("pageExists predicate drops slugs whose page is gone", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [
+        makeConcept("present", "injected"),
+        makeConcept("gone", "injected"),
+      ],
+    });
+    const turns = extractOracleTurns(getDb(), {
+      pageExists: (slug) => slug === "present",
+    });
+    expect(turns[0]?.groundTruthSlugs).toEqual(["present"]);
+  });
+
+  test("skips rows whose ground truth is empty after filtering", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [makeConcept("noinj", "not_injected")],
+    });
+    expect(extractOracleTurns(getDb()).length).toBe(0);
+  });
+
+  test("conversationIds filter restricts the scan", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      concepts: [makeConcept("a", "injected")],
+      createdAt: 100,
+    });
+    insertMessage("m2", "c2", 200);
+    insertLog({
+      conversationId: "c2",
+      messageId: "m2",
+      turn: 1,
+      mode: "router",
+      concepts: [makeConcept("b", "injected")],
+      createdAt: 200,
+    });
+    const turns = extractOracleTurns(getDb(), { conversationIds: ["c2"] });
+    expect(turns.length).toBe(1);
+    expect(turns[0]?.conversationId).toBe("c2");
+  });
+});