@vellumai/assistant 0.8.3 → 0.8.5

package/src/memory/v2/__tests__/harness-replay-input.test.ts

@@ -0,0 +1,225 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({ memory: { enabled: false } }),
+}));
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import type { OracleTurn } from "../harness/oracle.js";
+import { reconstructInput } from "../harness/replay-input.js";
+
+initializeDb();
+
+// loadNowText reads workspace files; a nonexistent dir yields "".
+const WORKSPACE = "/tmp/harness-replay-nonexistent-workspace";
+
+const ZERO_CONFIG = {
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+};
+
+let seq = 0;
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function config(historicalPairs: number): AssistantConfig {
+  return {
+    memory: { v2: { router: { historical_pairs: historicalPairs } } },
+  } as unknown as AssistantConfig;
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  role: string,
+  text: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role,
+      content: JSON.stringify([{ type: "text", text }]),
+      createdAt,
+    })
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertRouterLog(
+  conversationId: string,
+  messageId: string,
+  turn: number,
+  concepts: MemoryV2ConceptRowRecord[],
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId,
+      messageId,
+      turn,
+      mode: "router",
+      conceptsJson: JSON.stringify(concepts),
+      skillsJson: "[]",
+      configJson: JSON.stringify(ZERO_CONFIG),
+      createdAt,
+    })
+    .run();
+}
+
+function turnFor(
+  conversationId: string,
+  turn: number,
+  anchorMessageId: string,
+  anchorCreatedAt: number,
+): OracleTurn {
+  return {
+    conversationId,
+    turn,
+    anchorMessageId,
+    anchorCreatedAt,
+    groundTruthSlugs: ["x"],
+    loggedConfig: ZERO_CONFIG,
+    createdAt: anchorCreatedAt,
+  };
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+}
+
+describe("harness/replay-input reconstructInput", () => {
+  beforeEach(reset);
+
+  test("reconstructs the last (assistant,user) pair before the anchor reply", async () => {
+    insertMessage("u1", "c1", "user", "first user", 10);
+    insertMessage("a1", "c1", "assistant", "first reply", 20);
+    insertMessage("u2", "c1", "user", "second user", 30);
+    insertMessage("a2", "c1", "assistant", "second reply", 40); // anchor
+
+    const r = await reconstructInput(
+      getDb(),
+      turnFor("c1", 2, "a2", 40),
+      config(1),
+      WORKSPACE,
+    );
+    expect(r).not.toBeNull();
+    expect(r?.input.recentTurnPairs).toEqual([
+      { assistantMessage: "first reply", userMessage: "second user" },
+    ]);
+    expect(r?.meta.pairsReconstructed).toBe(1);
+    expect(r?.input.nowText).toBe("");
+  });
+
+  test("historical_pairs=2 returns the last two pairs", async () => {
+    insertMessage("u1", "c1", "user", "u-one", 10);
+    insertMessage("a1", "c1", "assistant", "a-one", 20);
+    insertMessage("u2", "c1", "user", "u-two", 30);
+    insertMessage("a2", "c1", "assistant", "a-two", 40);
+    insertMessage("u3", "c1", "user", "u-three", 50);
+    insertMessage("a3", "c1", "assistant", "a-three", 60); // anchor
+
+    const r = await reconstructInput(
+      getDb(),
+      turnFor("c1", 3, "a3", 60),
+      config(2),
+      WORKSPACE,
+    );
+    expect(r?.input.recentTurnPairs).toEqual([
+      { assistantMessage: "a-one", userMessage: "u-two" },
+      { assistantMessage: "a-two", userMessage: "u-three" },
+    ]);
+  });
+
+  test("returns null when the anchor message is not found", async () => {
+    insertMessage("u1", "c1", "user", "hi", 10);
+    const r = await reconstructInput(
+      getDb(),
+      turnFor("c1", 1, "missing-anchor", 999),
+      config(1),
+      WORKSPACE,
+    );
+    expect(r).toBeNull();
+  });
+
+  test("priorEverInjected unions injected/in_context slugs from earlier router turns", async () => {
+    insertMessage("u1", "c1", "user", "u1", 10);
+    insertMessage("a1", "c1", "assistant", "a1", 20);
+    insertMessage("u2", "c1", "user", "u2", 30);
+    insertMessage("a2", "c1", "assistant", "a2", 40); // anchor for turn 2
+    insertRouterLog(
+      "c1",
+      "a1",
+      1,
+      [
+        makeConcept("p1", "injected"),
+        makeConcept("p2", "in_context"),
+        makeConcept("p3", "not_injected"),
+      ],
+      20,
+    );
+
+    const r = await reconstructInput(
+      getDb(),
+      turnFor("c1", 2, "a2", 40),
+      config(1),
+      WORKSPACE,
+    );
+    const slugs = (r?.input.priorEverInjected ?? []).map((e) => e.slug).sort();
+    expect(slugs).toEqual(["p1", "p2"]);
+    expect(r?.meta.priorEverInjectedCount).toBe(2);
+  });
+});

package/src/memory/v2/__tests__/harness-runner.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, test } from "bun:test";
+
+import type { AssistantConfig } from "../../../config/types.js";
+import type { OracleTurn } from "../harness/oracle.js";
+import type { ReconstructedInput } from "../harness/replay-input.js";
+import type {
+  RetrievalInput,
+  RetrievalOutput,
+  Retriever,
+} from "../harness/retriever.js";
+import { runComparison } from "../harness/runner.js";
+
+const ZERO_CONFIG = {
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+};
+
+function oracleTurn(
+  conversationId: string,
+  turn: number,
+  groundTruthSlugs: string[],
+): OracleTurn {
+  return {
+    conversationId,
+    turn,
+    anchorMessageId: `${conversationId}-m${turn}`,
+    anchorCreatedAt: turn * 100,
+    groundTruthSlugs,
+    loggedConfig: ZERO_CONFIG,
+    createdAt: turn * 100,
+  };
+}
+
+const STUB_INPUT: RetrievalInput = {
+  workspaceDir: "/tmp/ws",
+  recentTurnPairs: [{ assistantMessage: "", userMessage: "hi" }],
+  nowText: "",
+  priorEverInjected: [],
+  config: {} as unknown as AssistantConfig,
+};
+
+const STUB_RECONSTRUCTED: ReconstructedInput = {
+  input: STUB_INPUT,
+  meta: {
+    windowPairs: 1,
+    pairsReconstructed: 1,
+    priorEverInjectedCount: 0,
+    nowReconstructedFromCurrent: true,
+  },
+};
+
+function fixedRetriever(name: string, selected: string[]): Retriever {
+  return {
+    name,
+    retrieve: async (): Promise<RetrievalOutput> => ({
+      selectedSlugs: selected,
+      sourceBySlug: new Map(selected.map((s): [string, string] => [s, name])),
+    }),
+  };
+}
+
+describe("harness/runner runComparison", () => {
+  test("scores each retriever against ground truth across turns", async () => {
+    const report = await runComparison({
+      retrievers: [
+        fixedRetriever("router", ["a", "b"]),
+        fixedRetriever("loop", ["a", "c"]),
+      ],
+      oracleTurns: [
+        oracleTurn("c1", 1, ["a", "b"]),
+        oracleTurn("c1", 2, ["a"]),
+      ],
+      reconstruct: async () => STUB_RECONSTRUCTED,
+      ks: [5],
+    });
+
+    expect(report.turnsConsidered).toBe(2);
+    expect(report.turnsScored).toBe(2);
+    expect(report.turnsSkipped).toBe(0);
+
+    const router = report.retrievers.find((r) => r.name === "router");
+    const loop = report.retrievers.find((r) => r.name === "loop");
+    // router recovers all ground truth on both turns → mean recall 1
+    expect(router?.aggregate.meanRecallAtK[5]).toBeCloseTo(1);
+    // loop: turn 1 = 1/2, turn 2 = 1/1 → mean 0.75
+    expect(loop?.aggregate.meanRecallAtK[5]).toBeCloseTo(0.75);
+  });
+
+  test("skips turns whose reconstruction returns null", async () => {
+    const report = await runComparison({
+      retrievers: [fixedRetriever("router", ["a"])],
+      oracleTurns: [oracleTurn("c1", 1, ["a"]), oracleTurn("c1", 2, ["a"])],
+      reconstruct: async (turn) =>
+        turn.turn === 2 ? null : STUB_RECONSTRUCTED,
+      ks: [5],
+    });
+
+    expect(report.turnsScored).toBe(1);
+    expect(report.turnsSkipped).toBe(1);
+    expect(report.perTurn.length).toBe(1);
+    expect(report.perTurn[0]?.turn).toBe(1);
+  });
+});

package/src/memory/v2/__tests__/injection-events.test.ts

@@ -0,0 +1,318 @@
+/**
+ * Tests for `assistant/src/memory/v2/injection-events.ts` and its sibling
+ * migration `256-memory-v2-injection-events.ts`.
+ *
+ * Coverage matrix:
+ *   - Migration creates the table + both indexes; safe to re-run.
+ *   - Backfill replays router-sourced concepts from memory_v2_activation_logs
+ *     and is idempotent on a forced re-run with cleared checkpoint.
+ *   - Backfill is a no-op when the activation-logs table doesn't exist
+ *     (pre-234 DB).
+ *   - recordInjectionEvents appends one row per slug per call; empty list
+ *     is a no-op.
+ *   - computeInjectionScore matches the closed-form decay at known deltas
+ *     (0d ≈ 1, 3d ≈ 0.5, 6d ≈ 0.25) and sums multiple events linearly.
+ *   - computeInjectionScores returns the same per-slug values in batch and
+ *     omits slugs with no events.
+ *
+ * Uses an in-memory bun:sqlite database — no real workspace DB.
+ */
+
+import { Database } from "bun:sqlite";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { makeMockLogger } from "../../../__tests__/helpers/mock-logger.js";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+import type { DrizzleDb } from "../../db-connection.js";
+import { getSqliteFrom } from "../../db-connection.js";
+import { migrateMemoryV2ActivationLogs } from "../../migrations/234-memory-v2-activation-logs.js";
+import {
+  downMemoryV2InjectionEvents,
+  migrateMemoryV2InjectionEvents,
+} from "../../migrations/256-memory-v2-injection-events.js";
+import * as schema from "../../schema.js";
+import {
+  computeInjectionScore,
+  computeInjectionScores,
+  INJECTION_SCORE_HALF_LIFE_MS,
+  recordInjectionEvents,
+} from "../injection-events.js";
+
+// memory_checkpoints is required by withCrashRecovery and is normally
+// created by an early core migration. Stand it up by hand so we can run
+// the v2 migrations in isolation against a fresh in-memory DB.
+const CHECKPOINTS_DDL = /*sql*/ `
+  CREATE TABLE memory_checkpoints (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL,
+    updated_at INTEGER NOT NULL
+  )
+`;
+
+let sqlite: Database;
+let database: DrizzleDb;
+
+beforeEach(() => {
+  sqlite = new Database(":memory:");
+  database = drizzle(sqlite, { schema });
+  getSqliteFrom(database).exec(CHECKPOINTS_DDL);
+});
+
+afterEach(() => {
+  sqlite.close();
+});
+
+function insertActivationLog(
+  rawDb: Database,
+  args: {
+    id: string;
+    concepts: Array<{ slug: string; source: string; status?: string }>;
+    createdAt: number;
+  },
+): void {
+  rawDb
+    .prepare(
+      `INSERT INTO memory_v2_activation_logs (
+        id, conversation_id, message_id, turn, mode,
+        concepts_json, skills_json, config_json, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      args.id,
+      "conv-1",
+      `msg-${args.id}`,
+      1,
+      "router",
+      JSON.stringify(args.concepts),
+      "[]",
+      "{}",
+      args.createdAt,
+    );
+}
+
+describe("migrateMemoryV2InjectionEvents", () => {
+  test("creates table and both indexes; safe to re-run", () => {
+    migrateMemoryV2InjectionEvents(database);
+    migrateMemoryV2InjectionEvents(database);
+
+    const raw = getSqliteFrom(database);
+    const table = raw
+      .query(
+        `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_v2_injection_events'`,
+      )
+      .get();
+    expect(table).toBeTruthy();
+
+    const indexNames = new Set(
+      (
+        raw
+          .query(
+            `SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='memory_v2_injection_events'`,
+          )
+          .all() as Array<{ name: string }>
+      ).map((r) => r.name),
+    );
+    expect(indexNames.has("idx_memory_v2_injection_events_slug_time")).toBe(
+      true,
+    );
+    expect(indexNames.has("idx_memory_v2_injection_events_time")).toBe(true);
+  });
+
+  test("backfill replays router-sourced concepts and ignores carry_over", () => {
+    migrateMemoryV2ActivationLogs(database);
+    const raw = getSqliteFrom(database);
+    insertActivationLog(raw, {
+      id: "log-1",
+      concepts: [
+        { slug: "alice", source: "router", status: "injected" },
+        { slug: "bob", source: "router", status: "in_context" },
+        { slug: "ghost", source: "carry_over", status: "not_injected" },
+      ],
+      createdAt: 1_000_000,
+    });
+    insertActivationLog(raw, {
+      id: "log-2",
+      concepts: [{ slug: "alice", source: "router", status: "injected" }],
+      createdAt: 2_000_000,
+    });
+
+    migrateMemoryV2InjectionEvents(database);
+
+    const rows = raw
+      .query(
+        `SELECT slug, injected_at FROM memory_v2_injection_events ORDER BY injected_at, slug`,
+      )
+      .all() as Array<{ slug: string; injected_at: number }>;
+    expect(rows).toEqual([
+      { slug: "alice", injected_at: 1_000_000 },
+      { slug: "bob", injected_at: 1_000_000 },
+      { slug: "alice", injected_at: 2_000_000 },
+    ]);
+  });
+
+  test("backfill is a no-op when memory_v2_activation_logs is absent", () => {
+    // No activation-logs migration applied first.
+    expect(() => migrateMemoryV2InjectionEvents(database)).not.toThrow();
+    const { n } = getSqliteFrom(database)
+      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+      .get() as { n: number };
+    expect(n).toBe(0);
+  });
+
+  test("forced re-run does not double-insert existing events", () => {
+    migrateMemoryV2ActivationLogs(database);
+    const raw = getSqliteFrom(database);
+    insertActivationLog(raw, {
+      id: "log-1",
+      concepts: [{ slug: "alice", source: "router", status: "injected" }],
+      createdAt: 1_000_000,
+    });
+
+    migrateMemoryV2InjectionEvents(database);
+    // Simulate someone manually clearing the checkpoint — the in-table
+    // guard should still prevent re-backfill.
+    raw
+      .prepare(
+        `DELETE FROM memory_checkpoints WHERE key = 'migration_memory_v2_injection_events_v1'`,
+      )
+      .run();
+    migrateMemoryV2InjectionEvents(database);
+
+    const { n } = raw
+      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+      .get() as { n: number };
+    expect(n).toBe(1);
+  });
+
+  test("downMemoryV2InjectionEvents drops the table", () => {
+    migrateMemoryV2InjectionEvents(database);
+    downMemoryV2InjectionEvents(database);
+    const table = getSqliteFrom(database)
+      .query(
+        `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_v2_injection_events'`,
+      )
+      .get();
+    expect(table).toBeFalsy();
+  });
+});
+
+describe("recordInjectionEvents", () => {
+  beforeEach(() => {
+    migrateMemoryV2InjectionEvents(database);
+  });
+
+  test("appends one row per slug at the same timestamp", () => {
+    const t = 1_000_000;
+    recordInjectionEvents(database, ["alice", "bob", "alice"], t);
+    const rows = getSqliteFrom(database)
+      .query(
+        `SELECT slug, injected_at FROM memory_v2_injection_events ORDER BY id`,
+      )
+      .all();
+    expect(rows).toEqual([
+      { slug: "alice", injected_at: t },
+      { slug: "bob", injected_at: t },
+      { slug: "alice", injected_at: t },
+    ]);
+  });
+
+  test("empty list is a no-op", () => {
+    recordInjectionEvents(database, [], 1_000_000);
+    const { n } = getSqliteFrom(database)
+      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+      .get() as { n: number };
+    expect(n).toBe(0);
+  });
+});
+
+describe("computeInjectionScore", () => {
+  beforeEach(() => {
+    migrateMemoryV2InjectionEvents(database);
+  });
+
+  test("returns 0 for a slug with no events", () => {
+    expect(computeInjectionScore(database, "missing", Date.now())).toBe(0);
+  });
+
+  test("single event 0 days ago → score ≈ 1", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(database, ["alice"], now);
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(1, 5);
+  });
+
+  test("single event 3 days (one half-life) ago → score ≈ 0.5", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(0.5, 5);
+  });
+
+  test("single event 6 days (two half-lives) ago → score ≈ 0.25", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - 2 * INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(0.25, 5);
+  });
+
+  test("multiple events sum independently", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(database, ["alice"], now);
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - 2 * INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(1.75, 5);
+  });
+});
+
+describe("computeInjectionScores", () => {
+  beforeEach(() => {
+    migrateMemoryV2InjectionEvents(database);
+  });
+
+  test("returns the same per-slug values as the single-slug helper", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(database, ["alice", "bob"], now);
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - INJECTION_SCORE_HALF_LIFE_MS,
+    );
+
+    const scores = computeInjectionScores(
+      database,
+      ["alice", "bob", "ghost"],
+      now,
+    );
+    expect(scores.get("alice")).toBeCloseTo(1.5, 5);
+    expect(scores.get("bob")).toBeCloseTo(1, 5);
+    // ghost has no events — omitted from the result, not present as 0.
+    expect(scores.has("ghost")).toBe(false);
+    expect(scores.get("alice")).toBeCloseTo(
+      computeInjectionScore(database, "alice", now),
+      5,
+    );
+  });
+
+  test("empty slug list returns empty map", () => {
+    expect(computeInjectionScores(database, [], Date.now()).size).toBe(0);
+  });
+});