@vellumai/assistant 0.8.4 → 0.8.6

package/src/memory/migrations/263-memory-v3-auto-edges.ts

@@ -0,0 +1,50 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+import { withCrashRecovery } from "./validate-migration-state.js";
+
+const CHECKPOINT_KEY = "migration_memory_v3_auto_edges_v1";
+
+/**
+ * Create the memory_v3_auto_edges table — the **learned** edge graph, a
+ * distinct class from the curated `edges:` frontmatter graph.
+ *
+ * Each row is a weighted directed association `source_slug → target_slug` that
+ * the edge-learning job (`memory_v3_edge_learning`) accrues from *used*
+ * co-activations (migration 262's `memory_v3_coactivation` rows) and decays
+ * over time. `weight` is a multiplicatively-decaying real; `last_reinforced_at`
+ * is the wall-clock ms of the most recent reinforcement, used by the decay
+ * pass to compute elapsed time per pair.
+ *
+ * Auto-edges are advisory: the read path consumes only above-threshold pairs
+ * via edge-expansion's `extraAdjacency` seam, and high-weight pairs surface as
+ * promotion *candidates* for the assistant to ratify into curated `edges:`
+ * during consolidation. This table never auto-writes page frontmatter.
+ *
+ * `PRIMARY KEY(source_slug, target_slug)` makes each ordered pair unique, so
+ * reinforce is a single UPSERT. The index on `(weight)` keeps the
+ * above-threshold scan and top-N promotion-candidate read cheap as the learned
+ * graph grows.
+ */
+export function migrateMemoryV3AutoEdges(database: DrizzleDb): void {
+  withCrashRecovery(database, CHECKPOINT_KEY, () => {
+    const raw = getSqliteFrom(database);
+    raw.exec(/*sql*/ `
+      CREATE TABLE IF NOT EXISTS memory_v3_auto_edges (
+        source_slug TEXT NOT NULL,
+        target_slug TEXT NOT NULL,
+        weight REAL NOT NULL,
+        last_reinforced_at INTEGER NOT NULL,
+        PRIMARY KEY (source_slug, target_slug)
+      )
+    `);
+    raw.exec(/*sql*/ `
+      CREATE INDEX IF NOT EXISTS idx_memory_v3_auto_edges_weight
+        ON memory_v3_auto_edges (weight)
+    `);
+  });
+}
+
+export function downMemoryV3AutoEdges(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `DROP TABLE IF EXISTS memory_v3_auto_edges`);
+}

package/src/memory/migrations/264-llm-request-log-call-site.ts

@@ -0,0 +1,29 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Adds `call_site` (nullable TEXT) to the `llm_request_logs` table.
+ *
+ * Records the logical call site that produced the row — `mainAgent`,
+ * `compactionAgent`, etc. (string values from `LLMCallSite` in
+ * `config/schemas/llm.ts`). Lets the LLM Context Inspector and other
+ * observability tools filter "show me only compaction calls" without
+ * having to infer from request payload shape.
+ *
+ * No backfill — historical rows stay NULL ("we don't know"). New rows
+ * stamped at insertion time by callers of `recordRequestLog`.
+ *
+ * Idempotent — re-running is a no-op once the column exists. Modeled on
+ * migration 252 (`llm-request-log-agent-loop-exit-reason`).
+ */
+export function migrateLlmRequestLogCallSite(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const columns = raw
+    .query(`PRAGMA table_info(llm_request_logs)`)
+    .all() as Array<{ name: string }>;
+  const columnNames = new Set(columns.map((c) => c.name));
+
+  if (!columnNames.has("call_site")) {
+    raw.exec(`ALTER TABLE llm_request_logs ADD COLUMN call_site TEXT`);
+  }
+}

package/src/memory/migrations/265-drop-provider-connection-status.ts

@@ -0,0 +1,26 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Drops the `status` column from the `provider_connections` table.
+ *
+ * Connection status ("active" | "disabled") has been removed — connections
+ * either exist or they don't. Profile status (enabled/disabled) is a separate
+ * concept and is untouched by this migration.
+ *
+ * Idempotent: checks PRAGMA table_info before issuing the DROP so re-running
+ * on a database that already lacks the column is a no-op.
+ *
+ * SQLite 3.35.0+ (Bun bundles a recent SQLite) supports ALTER TABLE DROP COLUMN.
+ */
+export function migrateDropProviderConnectionStatus(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const columns = raw
+    .query(`PRAGMA table_info(provider_connections)`)
+    .all() as Array<{ name: string }>;
+  const columnNames = new Set(columns.map((c) => c.name));
+
+  if (columnNames.has("status")) {
+    raw.exec(`ALTER TABLE provider_connections DROP COLUMN status`);
+  }
+}

package/src/memory/migrations/266-messages-client-message-id.ts

@@ -0,0 +1,43 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Adds `client_message_id` (nullable TEXT) to the `messages` table and
+ * creates a partial unique index on `(conversation_id, client_message_id)`
+ * for rows where `client_message_id IS NOT NULL`.
+ *
+ * The client-generated correlation nonce was previously wire-protocol-only
+ * (echoed on SSE `user_message_echo` events for optimistic-row dedup) but
+ * never persisted. Storing it enables server-side idempotency: a duplicate
+ * INSERT with the same `(conversation_id, client_message_id)` pair is
+ * silently skipped via `ON CONFLICT DO NOTHING`.
+ *
+ * Idempotent — re-running is a no-op once the column and index exist.
+ */
+export function migrateMessagesClientMessageId(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const columns = raw.query(`PRAGMA table_info(messages)`).all() as Array<{
+    name: string;
+  }>;
+  const columnNames = new Set(columns.map((c) => c.name));
+
+  if (!columnNames.has("client_message_id")) {
+    raw.exec(`ALTER TABLE messages ADD COLUMN client_message_id TEXT`);
+  }
+
+  // Partial unique index: only enforced when client_message_id is non-NULL.
+  // Messages without a client_message_id (assistant messages, system-generated
+  // messages, legacy rows) are unconstrained.
+  const indexes = raw.query(`PRAGMA index_list(messages)`).all() as Array<{
+    name: string;
+  }>;
+  const indexNames = new Set(indexes.map((i) => i.name));
+
+  if (!indexNames.has("idx_messages_conv_client_msg_id")) {
+    raw.exec(
+      `CREATE UNIQUE INDEX idx_messages_conv_client_msg_id
+       ON messages (conversation_id, client_message_id)
+       WHERE client_message_id IS NOT NULL`,
+    );
+  }
+}

package/src/memory/migrations/index.ts

@@ -234,6 +234,25 @@ export {
   downConversationCleanedAt,
   migrateConversationCleanedAt,
 } from "./259-conversation-cleaned-at.js";
+export {
+  downRenameCleanedAt,
+  migrateRenameCleanedAt,
+} from "./260-rename-cleaned-at.js";
+export {
+  downLlmUsageAddRawUsage,
+  migrateLlmUsageAddRawUsage,
+} from "./261-llm-usage-add-raw-usage.js";
+export {
+  downMemoryV3Coactivation,
+  migrateMemoryV3Coactivation,
+} from "./262-memory-v3-coactivation.js";
+export {
+  downMemoryV3AutoEdges,
+  migrateMemoryV3AutoEdges,
+} from "./263-memory-v3-auto-edges.js";
+export { migrateLlmRequestLogCallSite } from "./264-llm-request-log-call-site.js";
+export { migrateDropProviderConnectionStatus } from "./265-drop-provider-connection-status.js";
+export { migrateMessagesClientMessageId } from "./266-messages-client-message-id.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/migrations/registry.ts

@@ -53,6 +53,10 @@ import { downA2ATasks } from "./251-a2a-tasks.js";
 import { downExternalConversationBindingChatName } from "./254-external-conversation-binding-chat-name.js";
 import { downMemoryV2InjectionEvents } from "./256-memory-v2-injection-events.js";
 import { downConversationCleanedAt } from "./259-conversation-cleaned-at.js";
+import { downRenameCleanedAt } from "./260-rename-cleaned-at.js";
+import { downLlmUsageAddRawUsage } from "./261-llm-usage-add-raw-usage.js";
+import { downMemoryV3Coactivation } from "./262-memory-v3-coactivation.js";
+import { downMemoryV3AutoEdges } from "./263-memory-v3-auto-edges.js";
 
 export interface MigrationRegistryEntry {
   /** The checkpoint key written to memory_checkpoints on completion. */
@@ -453,6 +457,35 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
       "Add cleaned_at timestamp to conversations so /clean survives reload and forks inherit conditionally on fork point",
     down: downConversationCleanedAt,
   },
+  {
+    key: "migration_rename_cleaned_at_v1",
+    version: 53,
+    dependsOn: ["migration_conversation_cleaned_at_v1"],
+    description:
+      "Rename conversations.cleaned_at to history_stripped_at; the marker now records any injection-strip event (including compaction), not just /clean",
+    down: downRenameCleanedAt,
+  },
+  {
+    key: "migration_llm_usage_add_raw_usage_v1",
+    version: 54,
+    description:
+      "Add raw_usage TEXT column to llm_usage_events for storing the provider's untouched usage block as JSON (Anthropic TTL breakdown, OpenAI prompt/completion token details, etc.) so downstream consumers can extract provider-specific detail without per-field schema changes",
+    down: downLlmUsageAddRawUsage,
+  },
+  {
+    key: "migration_memory_v3_coactivation_v1",
+    version: 55,
+    description:
+      "Create memory_v3_coactivation table — append-only log of pass-1 → pass-N co-activation pairs (gradient signal) emitted by the v3 retrieval loop and reconciled later by edge-learning",
+    down: downMemoryV3Coactivation,
+  },
+  {
+    key: "migration_memory_v3_auto_edges_v1",
+    version: 56,
+    description:
+      "Create memory_v3_auto_edges table — weighted, decaying learned association graph (distinct from curated edges:) accrued by the edge-learning job from used co-activations and consumed above-threshold by edge expansion",
+    down: downMemoryV3AutoEdges,
+  },
 ];
 
 export function getMaxMigrationVersion(): number {

package/src/memory/schema/conversations.ts

@@ -1,9 +1,11 @@
+import { sql } from "drizzle-orm";
 import {
   index,
   integer,
   real,
   sqliteTable,
   text,
+  uniqueIndex,
 } from "drizzle-orm/sqlite-core";
 
 export const conversations = sqliteTable(
@@ -21,7 +23,7 @@ export const conversations = sqliteTable(
       .notNull()
       .default(0),
     contextCompactedAt: integer("context_compacted_at"),
-    cleanedAt: integer("cleaned_at"),
+    historyStrippedAt: integer("history_stripped_at"),
     slackContextCompactionWatermarkTs: text(
       "slack_context_compaction_watermark_ts",
     ),
@@ -66,8 +68,14 @@ export const messages = sqliteTable(
     content: text("content").notNull(),
     createdAt: integer("created_at").notNull(),
     metadata: text("metadata"),
+    clientMessageId: text("client_message_id"),
   },
-  (table) => [index("idx_messages_conversation_id").on(table.conversationId)],
+  (table) => [
+    index("idx_messages_conversation_id").on(table.conversationId),
+    uniqueIndex("idx_messages_conv_client_msg_id")
+      .on(table.conversationId, table.clientMessageId)
+      .where(sql`client_message_id IS NOT NULL`),
+  ],
 );
 
 export const toolInvocations = sqliteTable(

package/src/memory/schema/inference.ts

@@ -15,7 +15,6 @@ export const providerConnections = sqliteTable(
     name: text("name").primaryKey(),
     provider: text("provider").notNull(),
     auth: text("auth").notNull(),
-    status: text("status").notNull().default("active"),
     label: text("label"),
     baseUrl: text("base_url"),
     models: text("models"),

package/src/memory/schema/infrastructure.ts

@@ -138,6 +138,16 @@ export const llmRequestLogs = sqliteTable(
     responsePayload: text("response_payload").notNull(),
     createdAt: integer("created_at").notNull(),
     agentLoopExitReason: text("agent_loop_exit_reason"),
+    /**
+     * Logical call site that produced this row — e.g. `mainAgent`,
+     * `compactionAgent`. Stored as free-form text rather than enum-bound
+     * so a new call site can ship without a schema bump, but in practice
+     * callers pass values from `LLMCallSite` (`config/schemas/llm.ts`).
+     *
+     * Historical rows (pre-migration 264) stay NULL — "we don't know"
+     * rather than guessing `mainAgent`.
+     */
+    callSite: text("call_site"),
   },
   (table) => [
     index("idx_llm_request_logs_message_id").on(table.messageId),
@@ -217,6 +227,17 @@ export const llmUsageEvents = sqliteTable(
     outputTokens: integer("output_tokens").notNull(),
     cacheCreationInputTokens: integer("cache_creation_input_tokens"),
     cacheReadInputTokens: integer("cache_read_input_tokens"),
+    /**
+     * The provider's untouched `usage` block, serialized as JSON. Anthropic
+     * nests the TTL breakdown under `cache_creation.ephemeral_{5m,1h}_input_tokens`;
+     * OpenAI nests cached-read details under `prompt_tokens_details`; both are
+     * just preserved verbatim so admin charts and downstream consumers can
+     * extract whatever provider-specific detail they need without a schema
+     * change every time a provider exposes a new field. `null` when the
+     * provider didn't return a usage block or for rows persisted before
+     * migration 260.
+     */
+    rawUsage: text("raw_usage"),
     estimatedCostUsd: real("estimated_cost_usd"),
     pricingStatus: text("pricing_status").notNull(),
     llmCallCount: integer("llm_call_count"),

package/src/memory/tool-usage-store.ts

@@ -2,6 +2,7 @@ import { count, desc, lt } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import { getLogger } from "../util/logger.js";
+import { runAsyncSqlite } from "./db-async-query.js";
 import { getDb } from "./db-connection.js";
 import { toolInvocations } from "./schema.js";
 
@@ -47,25 +48,52 @@ export function getRecentInvocations(limit: number) {
 const log = getLogger("audit-log");
 
 /**
- * Delete tool invocation records older than the specified number of days.
- * Returns the number of deleted records. Does nothing if retentionDays is 0.
+ * Delete tool invocation records older than the specified number of
+ * days. Returns the number of deleted records. Does nothing if
+ * `retentionDays` is zero, negative, or non-finite.
+ *
+ * The DELETE runs through {@link runAsyncSqlite}: when the host has a
+ * `sqlite3` CLI it executes in a subprocess and the daemon's main
+ * event loop stays responsive. On hosts without the CLI the
+ * abstraction falls back to in-process blocking execution — the same
+ * behaviour the daemon had before this abstraction existed.
  */
-export function rotateToolInvocations(retentionDays: number): number {
-  if (retentionDays <= 0) return 0;
+export async function rotateToolInvocations(
+  retentionDays: number,
+): Promise<number> {
+  if (!Number.isFinite(retentionDays) || retentionDays <= 0) return 0;
 
-  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  // Math.floor guarantees a plain integer literal in the inlined SQL
+  // below; no decimal, no exponent, no surprise characters.
+  const cutoffMs = Math.floor(
+    Date.now() - retentionDays * 24 * 60 * 60 * 1000,
+  );
   const db = getDb();
 
-  // Count before delete (Drizzle's .run() returns void on bun-sqlite)
+  // Count before delete so we can return + log the affected row count
+  // (`runAsyncSqlite` does not surface SQLite's `changes()` value).
   const [countRow] = db
     .select({ value: count() })
     .from(toolInvocations)
-    .where(lt(toolInvocations.createdAt, cutoff))
+    .where(lt(toolInvocations.createdAt, cutoffMs))
     .all();
   const toDelete = countRow?.value ?? 0;
   if (toDelete === 0) return 0;
 
-  db.delete(toolInvocations).where(lt(toolInvocations.createdAt, cutoff)).run();
+  // `runAsyncSqlite` takes a raw SQL string — sqlite3 CLI subprocesses
+  // see SQL on stdin without a binding layer. `cutoffMs` is a plain
+  // integer (see Math.floor above), so inlining it here is safe.
+  const result = await runAsyncSqlite(
+    `DELETE FROM tool_invocations WHERE created_at < ${cutoffMs}`,
+  );
+  if (!result.ok) {
+    log.error(
+      { error: result.error, backend: result.backend, toDelete },
+      "tool_invocations purge failed",
+    );
+    return 0;
+  }
+
   log.info(
     `Rotated ${toDelete} audit log entries older than ${retentionDays} day(s)`,
   );

package/src/memory/v2/__tests__/backfill-jobs.test.ts

@@ -202,7 +202,10 @@ afterAll(() => {
   rmSync(tmpWorkspace, { recursive: true, force: true });
 });
 
-const { getDb, resetDb } = await import("../../db-connection.js");
+const { getDb } = await import("../../db-connection.js");
+const { resetDbForTesting } = await import(
+  "../../../__tests__/db-test-helpers.js"
+);
 const { initializeDb } = await import("../../db-init.js");
 const { rawExec } = await import("../../raw-query.js");
 const { conversations, memoryJobs, messages } = await import("../../schema.js");
@@ -247,7 +250,7 @@ function makeJob(
 }
 
 beforeEach(() => {
-  resetDb();
+  resetDbForTesting();
   initializeDb();
   // The shared template-DB caching does not clear WAL state between tests,
   // so explicitly truncate every table this suite writes to. Without this,

package/src/memory/v2/__tests__/consolidation-job.test.ts

@@ -102,6 +102,7 @@ mock.module("../../jobs-store.js", () => ({
     nextJobIdCounter += 1;
     return `job-${nextJobIdCounter}`;
   },
+  isMemoryEnabled: () => true,
 }));
 
 // ── Workspace pin ───────────────────────────────────────────────────

package/src/memory/v2/__tests__/harness-compare.test.ts

@@ -0,0 +1,186 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({ memory: { enabled: false } }),
+}));
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import { runComparisonOverHistory } from "../harness/compare.js";
+import type { RetrievalOutput, Retriever } from "../harness/retriever.js";
+
+initializeDb();
+
+// loadNowText reads workspace files; a nonexistent dir yields "".
+const WORKSPACE = "/tmp/harness-compare-nonexistent-workspace";
+
+const ZERO_CONFIG = {
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+};
+
+let seq = 0;
+
+function config(historicalPairs: number): AssistantConfig {
+  return {
+    memory: { v2: { router: { historical_pairs: historicalPairs } } },
+  } as unknown as AssistantConfig;
+}
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  role: string,
+  text: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role,
+      content: JSON.stringify([{ type: "text", text }]),
+      createdAt,
+    })
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertRouterLog(
+  conversationId: string,
+  messageId: string,
+  turn: number,
+  concepts: MemoryV2ConceptRowRecord[],
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId,
+      messageId,
+      turn,
+      mode: "router",
+      conceptsJson: JSON.stringify(concepts),
+      skillsJson: "[]",
+      configJson: JSON.stringify(ZERO_CONFIG),
+      createdAt,
+    })
+    .run();
+}
+
+function stubRetriever(name: string, selected: string[]): Retriever {
+  return {
+    name,
+    retrieve: async (): Promise<RetrievalOutput> => ({
+      selectedSlugs: selected,
+      sourceBySlug: new Map(selected.map((s): [string, string] => [s, name])),
+    }),
+  };
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+}
+
+/** Seed one router turn: user msg, assistant anchor, and the logged picks. */
+function seedTurn(groundTruth: string[]): void {
+  insertMessage("u1", "c1", "user", "hello", 10);
+  insertMessage("a1", "c1", "assistant", "hi", 20); // anchor for turn 1
+  insertRouterLog(
+    "c1",
+    "a1",
+    1,
+    groundTruth.map((slug) => makeConcept(slug, "injected")),
+    20,
+  );
+}
+
+describe("harness/compare runComparisonOverHistory", () => {
+  beforeEach(reset);
+
+  test("scores a stub retriever against the logged ground truth", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await runComparisonOverHistory({
+      db: getDb(),
+      workspaceDir: WORKSPACE,
+      config: config(1),
+      retrievers: [stubRetriever("stub", ["p1", "z"])],
+      ks: [5],
+    });
+
+    expect(report.turnsConsidered).toBe(1);
+    expect(report.turnsScored).toBe(1);
+    expect(report.turnsSkipped).toBe(0);
+    // hit p1, miss p2 → recall@5 = 1/2
+    expect(report.retrievers[0]?.aggregate.meanRecallAtK[5]).toBeCloseTo(0.5);
+  });
+
+  test("pageExists narrows the ground truth set", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await runComparisonOverHistory({
+      db: getDb(),
+      workspaceDir: WORKSPACE,
+      config: config(1),
+      retrievers: [stubRetriever("stub", ["p1", "z"])],
+      ks: [5],
+      pageExists: (slug) => slug === "p1", // p2 no longer exists
+    });
+
+    // ground truth is just {p1}; stub selected p1 → recall 1.0
+    expect(report.retrievers[0]?.aggregate.meanRecallAtK[5]).toBeCloseTo(1);
+  });
+});

package/src/memory/v2/__tests__/harness-metrics.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, test } from "bun:test";
+
+import { aggregate, evalTurn, recallAtK } from "../harness/metrics.js";
+import type { RetrievalOutput } from "../harness/retriever.js";
+
+function out(
+  selected: string[],
+  sourceBySlug: Record<string, string> = {},
+): RetrievalOutput {
+  return {
+    selectedSlugs: selected,
+    sourceBySlug: new Map(Object.entries(sourceBySlug)),
+  };
+}
+
+describe("harness/metrics recallAtK", () => {
+  test("fraction of ground truth recovered within k", () => {
+    const gt = new Set(["a", "b", "c", "d"]);
+    expect(recallAtK(["a", "b", "x"], gt, 10)).toBeCloseTo(0.5);
+    // only the top-2 selections count toward recall@2
+    expect(recallAtK(["a", "b", "c", "d"], gt, 2)).toBeCloseTo(0.5);
+    expect(recallAtK(["a", "b", "c", "d"], gt, 10)).toBeCloseTo(1);
+  });
+
+  test("empty ground truth is vacuously complete (recall 1)", () => {
+    expect(recallAtK([], new Set<string>(), 5)).toBe(1);
+  });
+
+  test("duplicate selections cannot push recall above 1.0", () => {
+    // A retriever emitting the same slug twice must not double-count it.
+    expect(recallAtK(["a", "a"], new Set(["a"]), 10)).toBe(1);
+    // Duplicates inside the top-k window still count once.
+    const gt = new Set(["a", "b"]);
+    expect(recallAtK(["a", "a", "b"], gt, 10)).toBeCloseTo(1);
+    expect(recallAtK(["a", "a", "b"], gt, 2)).toBeCloseTo(0.5);
+  });
+});
+
+describe("harness/metrics evalTurn", () => {
+  test("hits / misses / extras and per-lane attribution", () => {
+    const e = evalTurn(
+      out(["a", "b", "z"], { a: "tree", b: "sparse", z: "dense" }),
+      ["a", "b", "c"],
+      [5],
+    );
+    expect(e.hits.sort()).toEqual(["a", "b"]);
+    expect(e.misses).toEqual(["c"]);
+    // selected but not in ground truth — reported as diff, not error
+    expect(e.extras).toEqual(["z"]);
+    expect(e.recallAtK[5]).toBeCloseTo(2 / 3);
+    expect(e.hitsByLane).toEqual({ tree: 1, sparse: 1 });
+  });
+
+  test("failureReason surfaces and recall is 0", () => {
+    const o: RetrievalOutput = {
+      selectedSlugs: [],
+      sourceBySlug: new Map(),
+      failureReason: "no_provider",
+    };
+    const e = evalTurn(o, ["a"], [5]);
+    expect(e.failureReason).toBe("no_provider");
+    expect(e.recallAtK[5]).toBe(0);
+  });
+});
+
+describe("harness/metrics aggregate", () => {
+  test("means recall@k and failure rate across turns", () => {
+    const ks = [5];
+    const t1 = evalTurn(out(["a"]), ["a"], ks); // recall 1
+    const t2 = evalTurn(out([]), ["a"], ks); // recall 0
+    const agg = aggregate([t1, t2], ks);
+    expect(agg.turns).toBe(2);
+    expect(agg.meanRecallAtK[5]).toBeCloseTo(0.5);
+    expect(agg.failureRate).toBe(0);
+  });
+
+  test("empty input is well-defined", () => {
+    const agg = aggregate([], [5]);
+    expect(agg.turns).toBe(0);
+    expect(agg.meanRecallAtK[5]).toBe(0);
+    expect(agg.failureRate).toBe(0);
+  });
+});