@vellumai/assistant 0.8.4 → 0.8.5

package/src/memory/v2/__tests__/router.test.ts

@@ -116,7 +116,7 @@ mock.module("../../../providers/provider-send-message.js", () => ({
 // them. No mock needed for `daemon/identity-helpers.js`; it tolerates a
 // missing IDENTITY.md by returning null.
 
-const { runRouter } = await import("../router.js");
+const { runRouter, applyHistoricalCharBudget } = await import("../router.js");
 const { getPageIndex, invalidatePageIndex } = await import("../page-index.js");
 const { writePage } = await import("../page-store.js");
 
@@ -220,6 +220,7 @@ function makeConfig(overrides?: {
   batchSize?: number | null;
   tier1Size?: number | null;
   tier2Size?: number | null;
+  historicalPairsMaxChars?: number | null;
 }) {
   return {
     memory: {
@@ -231,6 +232,8 @@ function makeConfig(overrides?: {
           batch_size: overrides?.batchSize ?? null,
           tier1_size: overrides?.tier1Size ?? null,
           tier2_size: overrides?.tier2Size ?? null,
+          historical_pairs_max_chars:
+            overrides?.historicalPairsMaxChars ?? null,
         },
       },
     },
@@ -238,8 +241,12 @@ function makeConfig(overrides?: {
 }
 
 const COMMON_PARAMS = {
-  userMessage: "What's on my plate today?",
-  assistantMessage: "Let me check your plan.",
+  recentTurnPairs: [
+    {
+      assistantMessage: "Let me check your plan.",
+      userMessage: "What's on my plate today?",
+    },
+  ],
   nowText: "2026-05-10 14:00 PT",
   priorEverInjected: [] as { slug: string; turn: number }[],
 };
@@ -418,6 +425,78 @@ describe("runRouter — successful tool_use", () => {
     expect(blockB.cache_control).toBeUndefined();
   });
 
+  test("runRouterBatch front-truncates the oldest <last_turn> message when the char budget is exceeded", async () => {
+    await writePage(workspaceDir, makePage("alpha", { summary: "A" }));
+    providerStub = makeProvider(toolUseResponse([1]));
+
+    const longAssistant = "A".repeat(2_000);
+    const longUser = "B".repeat(2_000);
+    const recentAssistant = "Short prior.";
+    const justArrived = "What's relevant?";
+
+    await runRouter({
+      workspaceDir,
+      recentTurnPairs: [
+        { assistantMessage: longAssistant, userMessage: longUser },
+        { assistantMessage: recentAssistant, userMessage: justArrived },
+      ],
+      nowText: "now",
+      priorEverInjected: [],
+      // Budget: just enough room for the most-recent pair plus the old user
+      // line in full, leaving a small slice for the very oldest assistant
+      // (which should be front-truncated with the `…` marker).
+      config: makeConfig({
+        historicalPairsMaxChars:
+          recentAssistant.length + justArrived.length + longUser.length + 50,
+      }),
+    });
+
+    const [call] = providerCalls;
+    const userMsg = call.messages[0];
+    const blockB = userMsg.content[1] as { text: string };
+
+    // The just-arrived user message and the prior assistant reply survive
+    // verbatim because they're newest in the walk.
+    expect(blockB.text).toContain(`[user]: ${justArrived}`);
+    expect(blockB.text).toContain(`[assistant]: ${recentAssistant}`);
+
+    // The older user message survives verbatim (next newest after the
+    // most-recent pair).
+    expect(blockB.text).toContain(`[user]: ${longUser}`);
+
+    // The oldest message in the walk (the older assistant) is
+    // front-truncated, so its rendered line starts with the `…` marker
+    // and ends with the suffix of the original text.
+    expect(blockB.text).toContain("[assistant]: …");
+    expect(blockB.text.endsWith(`A\n</last_turn>`)).toBe(false); // sanity
+    // The full untruncated long-assistant string must NOT appear.
+    expect(blockB.text.includes(longAssistant)).toBe(false);
+    // The TAIL of the long-assistant string SHOULD appear (kept from front-truncation).
+    expect(blockB.text).toContain(longAssistant.slice(-10));
+  });
+
+  test("null historical_pairs_max_chars renders pairs verbatim regardless of size", async () => {
+    await writePage(workspaceDir, makePage("alpha", { summary: "A" }));
+    providerStub = makeProvider(toolUseResponse([1]));
+
+    const huge = "X".repeat(5_000);
+    await runRouter({
+      workspaceDir,
+      recentTurnPairs: [
+        { assistantMessage: huge, userMessage: "just arrived" },
+      ],
+      nowText: "now",
+      priorEverInjected: [],
+      config: makeConfig(), // historical_pairs_max_chars: null
+    });
+
+    const [call] = providerCalls;
+    const blockB = call.messages[0].content[1] as { text: string };
+    expect(blockB.text).toContain(`[assistant]: ${huge}`);
+    expect(blockB.text).toContain("[user]: just arrived");
+    expect(blockB.text).not.toContain("…");
+  });
+
   test("de-duplicates repeated IDs from the model while preserving order", async () => {
     providerStub = makeProvider(toolUseResponse([2, 1, 2]));
 
@@ -1017,3 +1096,92 @@ describe("runRouter — tier 2 (highest EMA)", () => {
     expect(warned).toBe(true);
   });
 });
+
+// ---------------------------------------------------------------------------
+// applyHistoricalCharBudget — pure helper covering the cap semantics.
+// ---------------------------------------------------------------------------
+
+describe("applyHistoricalCharBudget", () => {
+  test("null budget is a no-op (returns a shallow copy)", () => {
+    const pairs = [
+      { assistantMessage: "older asst", userMessage: "older user" },
+      { assistantMessage: "newer asst", userMessage: "newer user" },
+    ];
+    const out = applyHistoricalCharBudget(pairs, null);
+    expect(out).toEqual(pairs);
+    // shallow copy — not the same array reference, so callers can mutate freely
+    expect(out).not.toBe(pairs);
+  });
+
+  test("budget that fits every message returns content unchanged", () => {
+    const pairs = [
+      { assistantMessage: "AA", userMessage: "UU" },
+      { assistantMessage: "BB", userMessage: "VV" },
+    ];
+    const total = "AA".length + "UU".length + "BB".length + "VV".length; // 8
+    const out = applyHistoricalCharBudget(pairs, total);
+    expect(out).toEqual(pairs);
+  });
+
+  test("front-truncates the oldest still-includable message when the cap is exceeded", () => {
+    // Newest user is 10 chars, newest assistant is 10, older user is 10,
+    // older assistant is 20. Budget 35 leaves remaining = 35 - 10 - 10 - 10 = 5
+    // for the older assistant; 5 - 1 marker char = 4 kept chars from the END.
+    const pairs = [
+      { assistantMessage: "ABCDEFGHIJKLMNOPQRST", userMessage: "old-user--" },
+      { assistantMessage: "abcdefghij", userMessage: "uvwxyzUVWX" },
+    ];
+    const out = applyHistoricalCharBudget(pairs, 35);
+    expect(out).toEqual([
+      { assistantMessage: "…QRST", userMessage: "old-user--" },
+      { assistantMessage: "abcdefghij", userMessage: "uvwxyzUVWX" },
+    ]);
+    // Sanity: total content chars equals the budget.
+    const totalChars = out.reduce(
+      (acc, p) => acc + p.assistantMessage.length + p.userMessage.length,
+      0,
+    );
+    expect(totalChars).toBe(35);
+  });
+
+  test("drops older pairs entirely when even their first message has no room", () => {
+    // Budget 20 fits the most-recent pair exactly (10 + 10 = 20) and leaves
+    // zero room for the older pair, which is dropped entirely.
+    const pairs = [
+      { assistantMessage: "OLD-ASST00", userMessage: "OLD-USER00" },
+      { assistantMessage: "NEW-ASST00", userMessage: "NEW-USER00" },
+    ];
+    const out = applyHistoricalCharBudget(pairs, 20);
+    expect(out).toEqual([
+      { assistantMessage: "NEW-ASST00", userMessage: "NEW-USER00" },
+    ]);
+  });
+
+  test("drops the older message of the current pair when the user line consumes the whole budget", () => {
+    // Budget 10 just barely covers the newest user (10 chars). The pair's
+    // own assistant message has no room and is dropped (left empty).
+    const pairs = [
+      { assistantMessage: "ASSISTANTX", userMessage: "USER-NEW10" },
+    ];
+    const out = applyHistoricalCharBudget(pairs, 10);
+    expect(out).toEqual([{ assistantMessage: "", userMessage: "USER-NEW10" }]);
+  });
+
+  test("non-positive budgets return an empty array (no message survives)", () => {
+    const pairs = [{ assistantMessage: "x", userMessage: "y" }];
+    expect(applyHistoricalCharBudget(pairs, 0)).toEqual(pairs);
+    // Negative budgets are degenerate but should not throw.
+    expect(applyHistoricalCharBudget(pairs, -5)).toEqual(pairs);
+  });
+
+  test("budget smaller than the truncation marker drops the would-truncate message", () => {
+    // Budget 11: covers full newest user (10 chars). Remaining 1 char is not
+    // enough room for the marker, so the next message (newest assistant)
+    // is dropped entirely rather than emitting a marker-only message.
+    const pairs = [
+      { assistantMessage: "ASSISTANTX", userMessage: "USER-NEW10" },
+    ];
+    const out = applyHistoricalCharBudget(pairs, 11);
+    expect(out).toEqual([{ assistantMessage: "", userMessage: "USER-NEW10" }]);
+  });
+});

package/src/memory/v2/harness/compare.ts

@@ -0,0 +1,57 @@
+/**
+ * Run the comparison harness over a sample of historical turns.
+ *
+ * Ties the harness pieces together: pull oracle turns from telemetry, run each
+ * retriever over each turn's reconstructed inputs, score against the logged
+ * ground truth. Kept separate from the route handler so it can be unit-tested
+ * with a stub retriever and a fixture DB — no live router / LLM.
+ */
+
+import type { AssistantConfig } from "../../../config/types.js";
+import type { DrizzleDb } from "../../db-connection.js";
+import { extractOracleTurns } from "./oracle.js";
+import { reconstructInput } from "./replay-input.js";
+import type { Retriever } from "./retriever.js";
+import { type ComparisonReport, runComparison } from "./runner.js";
+
+export interface RunComparisonOverHistoryParams {
+  db: DrizzleDb;
+  workspaceDir: string;
+  config: AssistantConfig;
+  retrievers: readonly Retriever[];
+  ks: number[];
+  limit?: number;
+  strategy?: "recent" | "random";
+  conversationIds?: string[];
+  includeNotInjected?: boolean;
+  pageExists?: (slug: string) => boolean;
+  signal?: AbortSignal;
+}
+
+export async function runComparisonOverHistory(
+  params: RunComparisonOverHistoryParams,
+): Promise<ComparisonReport> {
+  const { db, workspaceDir, config } = params;
+
+  const oracleTurns = extractOracleTurns(db, {
+    ...(params.limit !== undefined ? { limit: params.limit } : {}),
+    ...(params.strategy !== undefined ? { strategy: params.strategy } : {}),
+    ...(params.conversationIds !== undefined
+      ? { conversationIds: params.conversationIds }
+      : {}),
+    ...(params.includeNotInjected !== undefined
+      ? { includeNotInjected: params.includeNotInjected }
+      : {}),
+    ...(params.pageExists !== undefined
+      ? { pageExists: params.pageExists }
+      : {}),
+  });
+
+  return runComparison({
+    retrievers: params.retrievers,
+    oracleTurns,
+    reconstruct: (turn) => reconstructInput(db, turn, config, workspaceDir),
+    ks: params.ks,
+    ...(params.signal !== undefined ? { signal: params.signal } : {}),
+  });
+}

package/src/memory/v2/harness/metrics.ts

@@ -0,0 +1,124 @@
+/**
+ * Recall@k and per-lane diff for the comparison harness.
+ *
+ * Ground truth is the current router's logged selections (see `oracle.ts`). A
+ * retriever's "extras" (selected, not in ground truth) are reported as a
+ * *diff*, not an error — a better retriever may legitimately surface pages the
+ * router missed. recall@k is the primary signal.
+ */
+
+import type { RetrievalOutput } from "./retriever.js";
+
+export interface TurnEval {
+  groundTruth: string[];
+  selected: string[];
+  /** Ground-truth slugs the retriever selected (anywhere in its output). */
+  hits: string[];
+  /** Ground-truth slugs the retriever missed entirely. */
+  misses: string[];
+  /** Selected slugs not in ground truth — diff, not error. */
+  extras: string[];
+  /** recall@k for each requested k. */
+  recallAtK: Record<number, number>;
+  /** Counts of hits grouped by the retriever's source/lane labels. */
+  hitsByLane: Record<string, number>;
+  costUsd?: number;
+  failureReason: string | null;
+}
+
+export interface AggregateEval {
+  turns: number;
+  meanRecallAtK: Record<number, number>;
+  failureRate: number;
+  meanCostUsd?: number;
+}
+
+/**
+ * recall@k = |topK(selected) ∩ G| / |G|. An empty ground-truth set is defined
+ * as recall 1 (nothing to recall — vacuously complete).
+ */
+export function recallAtK(
+  selected: readonly string[],
+  groundTruth: ReadonlySet<string>,
+  k: number,
+): number {
+  if (groundTruth.size === 0) return 1;
+  let hit = 0;
+  for (const slug of selected.slice(0, k)) {
+    if (groundTruth.has(slug)) hit++;
+  }
+  return hit / groundTruth.size;
+}
+
+export function evalTurn(
+  output: RetrievalOutput,
+  groundTruth: readonly string[],
+  ks: readonly number[],
+): TurnEval {
+  const gtList = Array.from(new Set(groundTruth));
+  const gtSet = new Set(gtList);
+  const selectedSet = new Set(output.selectedSlugs);
+
+  const hits: string[] = [];
+  const misses: string[] = [];
+  for (const slug of gtList) {
+    (selectedSet.has(slug) ? hits : misses).push(slug);
+  }
+  const extras = output.selectedSlugs.filter((s) => !gtSet.has(s));
+
+  const recall: Record<number, number> = {};
+  for (const k of ks) {
+    recall[k] = recallAtK(output.selectedSlugs, gtSet, k);
+  }
+
+  const hitsByLane: Record<string, number> = {};
+  for (const slug of hits) {
+    const lane = output.sourceBySlug.get(slug) ?? "unknown";
+    hitsByLane[lane] = (hitsByLane[lane] ?? 0) + 1;
+  }
+
+  return {
+    groundTruth: gtList,
+    selected: output.selectedSlugs,
+    hits,
+    misses,
+    extras,
+    recallAtK: recall,
+    hitsByLane,
+    ...(output.cost?.usd !== undefined ? { costUsd: output.cost.usd } : {}),
+    failureReason: output.failureReason ?? null,
+  };
+}
+
+export function aggregate(
+  perTurn: readonly TurnEval[],
+  ks: readonly number[],
+): AggregateEval {
+  const turns = perTurn.length;
+
+  const meanRecallAtK: Record<number, number> = {};
+  for (const k of ks) {
+    if (turns === 0) {
+      meanRecallAtK[k] = 0;
+      continue;
+    }
+    let sum = 0;
+    for (const t of perTurn) sum += t.recallAtK[k] ?? 0;
+    meanRecallAtK[k] = sum / turns;
+  }
+
+  const failures = perTurn.filter((t) => t.failureReason != null).length;
+  const costed = perTurn.filter((t) => t.costUsd !== undefined);
+
+  return {
+    turns,
+    meanRecallAtK,
+    failureRate: turns === 0 ? 0 : failures / turns,
+    ...(costed.length > 0
+      ? {
+          meanCostUsd:
+            costed.reduce((s, t) => s + (t.costUsd ?? 0), 0) / costed.length,
+        }
+      : {}),
+  };
+}

package/src/memory/v2/harness/oracle.ts

@@ -0,0 +1,145 @@
+/**
+ * Oracle extraction — the current router's logged selections as silver-standard
+ * ground truth.
+ *
+ * Source: `memory_v2_activation_logs` rows with `mode = 'router'`. Each row's
+ * `messageId` is backfilled to the turn's assistant message (see
+ * `backfillMemoryV2ActivationMessageId`), so we join `messageId → messages.id`
+ * to anchor the turn — robust, no fragile turn-counting. Rows whose messageId
+ * is null (the in-flight turn) or no longer resolves are skipped.
+ *
+ * Ground truth G(turn) = selected slugs with status ∈ {injected, in_context}
+ * (what actually reached the model), optionally + not_injected, and — when a
+ * `pageExists` predicate is supplied — only slugs whose page still exists
+ * (neither retriever can find a nonexistent page). page_missing / corrupt are
+ * always excluded.
+ */
+
+import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm";
+
+import type { DrizzleDb } from "../../db-connection.js";
+import type {
+  MemoryV2ConceptRowRecord,
+  MemoryV2ConfigSnapshot,
+} from "../../memory-v2-activation-log-store.js";
+import { memoryV2ActivationLogs, messages } from "../../schema.js";
+
+export interface OracleTurn {
+  conversationId: string;
+  turn: number;
+  /** Backfilled assistant-message id for this turn — the reconstruction anchor. */
+  anchorMessageId: string;
+  /** `created_at` of the anchor message; reconstruction cuts strictly before it. */
+  anchorCreatedAt: number;
+  /** Slugs the router's judgment put in front of the model (the recall target). */
+  groundTruthSlugs: string[];
+  loggedConfig: MemoryV2ConfigSnapshot;
+  createdAt: number;
+}
+
+export interface ExtractOracleOptions {
+  /** Max log rows to scan (default 50). Some are skipped, so result ≤ limit. */
+  limit?: number;
+  strategy?: "recent" | "random";
+  conversationIds?: string[];
+  /** Include status "not_injected" (selected but cut by the cap) in G. Default false. */
+  includeNotInjected?: boolean;
+  /**
+   * Page-existence predicate, typically backed by `getPageIndex().bySlug`.
+   * When provided, ground-truth slugs whose page no longer exists are dropped.
+   * Omit in unit tests.
+   */
+  pageExists?: (slug: string) => boolean;
+}
+
+export function extractOracleTurns(
+  db: DrizzleDb,
+  options: ExtractOracleOptions = {},
+): OracleTurn[] {
+  const {
+    limit = 50,
+    strategy = "recent",
+    conversationIds,
+    includeNotInjected = false,
+    pageExists,
+  } = options;
+
+  const allowedStatuses = new Set<string>(["injected", "in_context"]);
+  if (includeNotInjected) allowedStatuses.add("not_injected");
+
+  const filters = [
+    eq(memoryV2ActivationLogs.mode, "router"),
+    isNotNull(memoryV2ActivationLogs.messageId),
+  ];
+  if (conversationIds && conversationIds.length > 0) {
+    filters.push(
+      inArray(memoryV2ActivationLogs.conversationId, conversationIds),
+    );
+  }
+
+  const rows = db
+    .select({
+      conversationId: memoryV2ActivationLogs.conversationId,
+      messageId: memoryV2ActivationLogs.messageId,
+      turn: memoryV2ActivationLogs.turn,
+      conceptsJson: memoryV2ActivationLogs.conceptsJson,
+      configJson: memoryV2ActivationLogs.configJson,
+      createdAt: memoryV2ActivationLogs.createdAt,
+    })
+    .from(memoryV2ActivationLogs)
+    .where(and(...filters))
+    .orderBy(
+      strategy === "random"
+        ? sql`RANDOM()`
+        : desc(memoryV2ActivationLogs.createdAt),
+    )
+    .limit(limit)
+    .all();
+
+  const turns: OracleTurn[] = [];
+  for (const row of rows) {
+    const messageId = row.messageId;
+    if (messageId == null) continue;
+
+    const anchor = db
+      .select({ createdAt: messages.createdAt })
+      .from(messages)
+      .where(eq(messages.id, messageId))
+      .limit(1)
+      .all();
+    const anchorRow = anchor[0];
+    if (!anchorRow) continue;
+
+    let concepts: MemoryV2ConceptRowRecord[];
+    let loggedConfig: MemoryV2ConfigSnapshot;
+    try {
+      concepts = JSON.parse(row.conceptsJson) as MemoryV2ConceptRowRecord[];
+      loggedConfig = JSON.parse(row.configJson) as MemoryV2ConfigSnapshot;
+    } catch {
+      continue;
+    }
+
+    const seen = new Set<string>();
+    const groundTruthSlugs: string[] = [];
+    for (const concept of concepts) {
+      if (!allowedStatuses.has(concept.status)) continue;
+      if (pageExists && !pageExists(concept.slug)) continue;
+      if (seen.has(concept.slug)) continue;
+      seen.add(concept.slug);
+      groundTruthSlugs.push(concept.slug);
+    }
+    if (groundTruthSlugs.length === 0) continue;
+
+    turns.push({
+      conversationId: row.conversationId,
+      turn: row.turn,
+      anchorMessageId: messageId,
+      anchorCreatedAt: anchorRow.createdAt,
+      groundTruthSlugs,
+      loggedConfig,
+      createdAt: row.createdAt,
+    });
+  }
+
+  return turns;
+}