@vellumai/assistant 0.8.3 → 0.8.5

package/src/memory/v2/harness/runner.ts

@@ -0,0 +1,106 @@
+/**
+ * Comparison runner — execute N retrievers over a set of oracle turns and score
+ * each against ground truth.
+ *
+ * The runner is DB/workspace-agnostic: input reconstruction is injected as a
+ * function, so it can be unit-tested with stubs and the route/CLI can wire in
+ * the real `reconstructInput` (which needs a DB + workspace).
+ */
+
+import {
+  aggregate,
+  type AggregateEval,
+  evalTurn,
+  type TurnEval,
+} from "./metrics.js";
+import type { OracleTurn } from "./oracle.js";
+import type { ReconstructedInput } from "./replay-input.js";
+import type { Retriever } from "./retriever.js";
+
+export interface ComparisonTurnResult {
+  conversationId: string;
+  turn: number;
+  /** Per-retriever evaluation for this turn, keyed by retriever name. */
+  byRetriever: Record<string, TurnEval>;
+}
+
+export interface RetrieverReport {
+  name: string;
+  aggregate: AggregateEval;
+}
+
+export interface ComparisonReport {
+  ks: number[];
+  /** Oracle turns handed to the runner. */
+  turnsConsidered: number;
+  /** Turns actually scored (reconstruction succeeded). */
+  turnsScored: number;
+  /** Turns skipped because input reconstruction returned null. */
+  turnsSkipped: number;
+  perTurn: ComparisonTurnResult[];
+  retrievers: RetrieverReport[];
+}
+
+export interface RunComparisonParams {
+  retrievers: readonly Retriever[];
+  oracleTurns: readonly OracleTurn[];
+  /** Reconstruct a turn's retriever input; return null to skip the turn. */
+  reconstruct: (turn: OracleTurn) => Promise<ReconstructedInput | null>;
+  ks: readonly number[];
+  signal?: AbortSignal;
+}
+
+export async function runComparison(
+  params: RunComparisonParams,
+): Promise<ComparisonReport> {
+  const { retrievers, oracleTurns, reconstruct, ks, signal } = params;
+
+  const perTurn: ComparisonTurnResult[] = [];
+  const perRetrieverEvals = new Map<string, TurnEval[]>();
+  for (const retriever of retrievers) {
+    perRetrieverEvals.set(retriever.name, []);
+  }
+
+  let turnsScored = 0;
+  let turnsSkipped = 0;
+
+  for (const turn of oracleTurns) {
+    if (signal?.aborted) break;
+
+    const reconstructed = await reconstruct(turn);
+    if (!reconstructed) {
+      turnsSkipped++;
+      continue;
+    }
+    turnsScored++;
+
+    const byRetriever: Record<string, TurnEval> = {};
+    for (const retriever of retrievers) {
+      if (signal?.aborted) break;
+      const output = await retriever.retrieve(reconstructed.input);
+      const turnEval = evalTurn(output, turn.groundTruthSlugs, ks);
+      byRetriever[retriever.name] = turnEval;
+      perRetrieverEvals.get(retriever.name)?.push(turnEval);
+    }
+
+    perTurn.push({
+      conversationId: turn.conversationId,
+      turn: turn.turn,
+      byRetriever,
+    });
+  }
+
+  const retrieverReports: RetrieverReport[] = retrievers.map((retriever) => ({
+    name: retriever.name,
+    aggregate: aggregate(perRetrieverEvals.get(retriever.name) ?? [], ks),
+  }));
+
+  return {
+    ks: [...ks],
+    turnsConsidered: oracleTurns.length,
+    turnsScored,
+    turnsSkipped,
+    perTurn,
+    retrievers: retrieverReports,
+  };
+}

package/src/memory/v2/harness/trace.ts

@@ -0,0 +1,58 @@
+/**
+ * Descent-trace schema for a tree-walking retriever.
+ *
+ * Defined ahead of its producer: the comparison harness renders this and a
+ * tree-walking retriever emits it; a tier-based retriever (no tree walk) leaves
+ * `RetrievalOutput.trace` undefined. Per level it records which branches were
+ * considered, descended, and skipped plus the model's reasoning, so a wrong
+ * high-level skip is observable rather than silent.
+ */
+
+import type { RetrievalCost } from "./retriever.js";
+
+/** A scout lane's contribution on one pass. */
+export interface ScoutResult {
+  lane: "hot" | "sparse" | "dense";
+  slugs: string[];
+  /** Optional per-slug score (BM25 / cosine / EMA) for inspection. */
+  scoreBySlug?: Record<string, number>;
+}
+
+/** One level of the tree walk: what was considered, descended, and skipped. */
+export interface TreeLevel {
+  /** Node whose index page was read ("" for root, else a branch path). */
+  node: string;
+  considered: string[];
+  descended: string[];
+  skipped: string[];
+  /** The model's stated reason for the descend/skip split at this node. */
+  reasoning: string;
+  cost?: RetrievalCost;
+}
+
+/** A 1–2 hop walk along the curated `edges:` graph from a seed page. */
+export interface EdgeExpansion {
+  from: string;
+  pulled: string[];
+}
+
+/** The gate's decision at the end of a pass. */
+export interface GateDecision {
+  decision: "ready" | "more";
+  /** When "more", the generated follow-up queries seeding the next pass. */
+  questions?: string[];
+}
+
+/** Everything that happened on one pass of the loop. */
+export interface DescentPass {
+  passNumber: number;
+  scouts?: ScoutResult[];
+  treeLevels?: TreeLevel[];
+  edgeExpansions?: EdgeExpansion[];
+  gate?: GateDecision;
+}
+
+/** A full loop execution, pass by pass. */
+export interface DescentTrace {
+  passes: DescentPass[];
+}

package/src/memory/v2/injection-events.ts

@@ -0,0 +1,101 @@
+import { getLogger } from "../../util/logger.js";
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+const log = getLogger("memory-v2-injection-events");
+
+/**
+ * Half-life of the injection-frequency decay, in milliseconds.
+ *
+ * Per the memory router v4 spec: a +1 from 3 days ago contributes 0.5; from
+ * 6 days ago 0.25. Decoupled from turn volume — busy and quiet days decay
+ * at the same wall-clock rate.
+ */
+export const INJECTION_SCORE_HALF_LIFE_MS = 3 * 24 * 60 * 60 * 1000;
+
+export const INJECTION_SCORE_LAMBDA =
+  Math.log(2) / INJECTION_SCORE_HALF_LIFE_MS;
+
+// Events past 6 half-lives contribute <1.6% each. Reads bound the scan to
+// this window so per-slug score computation stays cheap as history grows.
+const READ_WINDOW_MS = 6 * INJECTION_SCORE_HALF_LIFE_MS;
+
+function decayContribution(elapsedMs: number): number {
+  return Math.exp(-INJECTION_SCORE_LAMBDA * elapsedMs);
+}
+
+/**
+ * Append one event per slug. Best-effort — a SQLite write must never abort
+ * the agent turn on top of a successful routing decision the rest of the
+ * caller depends on.
+ */
+export function recordInjectionEvents(
+  database: DrizzleDb,
+  slugs: readonly string[],
+  injectedAt: number,
+): void {
+  if (slugs.length === 0) return;
+  try {
+    const raw = getSqliteFrom(database);
+    const insert = raw.prepare(
+      `INSERT INTO memory_v2_injection_events (slug, injected_at) VALUES (?, ?)`,
+    );
+    const append = raw.transaction((items: readonly string[]) => {
+      for (const slug of items) insert.run(slug, injectedAt);
+    });
+    append(slugs);
+  } catch (err) {
+    log.warn(
+      { err, slugCount: slugs.length },
+      "failed to record injection events; continuing",
+    );
+  }
+}
+
+/** `score(now) = Σᵢ exp(-λ × (now - tᵢ))` over events within READ_WINDOW_MS. */
+export function computeInjectionScore(
+  database: DrizzleDb,
+  slug: string,
+  now: number,
+): number {
+  const cutoff = now - READ_WINDOW_MS;
+  const raw = getSqliteFrom(database);
+  const rows = raw
+    .query(
+      `SELECT injected_at FROM memory_v2_injection_events
+        WHERE slug = ? AND injected_at >= ?`,
+    )
+    .all(slug, cutoff) as Array<{ injected_at: number }>;
+  let score = 0;
+  for (const row of rows) score += decayContribution(now - row.injected_at);
+  return score;
+}
+
+/**
+ * Batch variant of `computeInjectionScore` — single SQL pass scoped to the
+ * requested slugs so tier assignment doesn't issue O(M) queries. Slugs
+ * with no events in the read window are omitted from the result; callers
+ * should treat a missing entry as score 0.
+ */
+export function computeInjectionScores(
+  database: DrizzleDb,
+  slugs: readonly string[],
+  now: number,
+): Map<string, number> {
+  const out = new Map<string, number>();
+  if (slugs.length === 0) return out;
+  const cutoff = now - READ_WINDOW_MS;
+  const raw = getSqliteFrom(database);
+  const placeholders = slugs.map(() => "?").join(",");
+  const rows = raw
+    .query(
+      `SELECT slug, injected_at FROM memory_v2_injection_events
+        WHERE slug IN (${placeholders}) AND injected_at >= ?`,
+    )
+    .all(...slugs, cutoff) as Array<{ slug: string; injected_at: number }>;
+  for (const row of rows) {
+    const prev = out.get(row.slug) ?? 0;
+    out.set(row.slug, prev + decayContribution(now - row.injected_at));
+  }
+  return out;
+}

package/src/memory/v2/injection.ts

@@ -43,8 +43,9 @@ import {
   isCliCommandSlug,
 } from "./cli-command-store.js";
 import { getEdgeIndex } from "./edge-index.js";
+import { recordInjectionEvents } from "./injection-events.js";
 import { readPage, renderPageContent } from "./page-store.js";
-import { runRouter } from "./router.js";
+import { type RouterTurnPair, runRouter } from "./router.js";
 import { getSkillCapability, isSkillSlug } from "./skill-store.js";
 import type { ActivationState, EverInjectedEntry } from "./types.js";
 
@@ -80,10 +81,13 @@ export interface InjectMemoryV2BlockParams {
   conversationId: string;
   /** Caller-tracked turn number, persisted with each new everInjected entry. */
   currentTurn: number;
-  /** Latest user message text (the turn that triggered this call). */
-  userMessage: string;
-  /** Prior assistant message text (empty string at conversation start). */
-  assistantMessage: string;
+  /**
+   * Recent assistant/user turn pairs, oldest first. Must contain at least
+   * one entry whose `userMessage` is the just-arrived turn that triggered
+   * this call. The number of pairs the production caller passes is
+   * controlled by `memory.v2.router.historical_pairs`.
+   */
+  recentTurnPairs: readonly RouterTurnPair[];
   /** NOW context (autoloaded essentials/threads/recent or NOW.md). */
   nowText: string;
   /** Resolved messageId to persist on the activation_state row. */
@@ -139,14 +143,21 @@ export async function injectMemoryV2Block(
     database,
     conversationId,
     currentTurn,
-    userMessage,
-    assistantMessage,
+    recentTurnPairs,
     nowText,
     messageId,
     config,
     signal,
   } = params;
 
+  // The spreading-activation fallback (only used when the router is off)
+  // still needs the most recent (assistant, user) pair for semantic
+  // scoring. Pulling these from the last pair preserves bit-identical
+  // K=1 behavior — the router-off path never benefits from extra pairs.
+  const lastPair = recentTurnPairs[recentTurnPairs.length - 1];
+  const userMessage = lastPair.userMessage;
+  const assistantMessage = lastPair.assistantMessage;
+
   const workspaceDir = getWorkspaceDir();
   const mode: InjectMemoryV2Mode = params.mode ?? "per-turn";
 
@@ -173,8 +184,7 @@ export async function injectMemoryV2Block(
       database,
       conversationId,
       currentTurn,
-      userMessage,
-      assistantMessage,
+      recentTurnPairs,
       nowText,
       messageId,
       config,
@@ -520,8 +530,7 @@ async function injectViaRouter(args: {
   database: DrizzleDb;
   conversationId: string;
   currentTurn: number;
-  userMessage: string;
-  assistantMessage: string;
+  recentTurnPairs: readonly RouterTurnPair[];
   nowText: string;
   messageId: string;
   config: AssistantConfig;
@@ -533,8 +542,7 @@ async function injectViaRouter(args: {
     database,
     conversationId,
     currentTurn,
-    userMessage,
-    assistantMessage,
+    recentTurnPairs,
     nowText,
     messageId,
     config,
@@ -547,14 +555,23 @@ async function injectViaRouter(args: {
 
   const routerResult = await runRouter({
     workspaceDir,
-    userMessage,
-    assistantMessage,
+    recentTurnPairs,
     nowText,
     priorEverInjected,
     config,
+    database,
     ...(signal ? { signal } : {}),
   });
 
+  // Record router selections to the EMA event log. The router decided these
+  // slugs are relevant THIS turn (regardless of whether they're newly
+  // rendered or re-picked from prior context). The helper swallows its own
+  // errors — a SQLite write must not abort the turn on top of a successful
+  // routing decision the rest of this function depends on.
+  if (routerResult.failureReason === null) {
+    recordInjectionEvents(database, routerResult.selectedSlugs, Date.now());
+  }
+
   if (routerResult.failureReason !== null) {
     log.warn(
       { failureReason: routerResult.failureReason },
@@ -603,12 +620,12 @@ async function injectViaRouter(args: {
 
   // Build minimal telemetry rows for the union of router-selected slugs and
   // prior `everInjected` slugs. Router-mode rows zero out every activation
-  // value (no spreading activation runs). Slugs the router picked this turn
-  // get `source: "router"`; prior-everInjected slugs the router did NOT
-  // re-pick get `source: "carry_over"`. The `status` placeholder is
-  // overwritten by `finalizeInjection`.
-  const routerPicked = new Set(routerResult.selectedSlugs);
-  const telemetrySlugs = new Set<string>(routerPicked);
+  // value (no spreading activation runs). Slugs the router picked carry
+  // their batch's tier tag from `routerResult.sourceBySlug` (e.g. `tier1`,
+  // `tier3:2`); prior-everInjected slugs the router did NOT re-pick get
+  // `source: "carry_over"`. The `status` placeholder is overwritten by
+  // `finalizeInjection`.
+  const telemetrySlugs = new Set<string>(routerResult.selectedSlugs);
   for (const entry of priorEverInjected) telemetrySlugs.add(entry.slug);
   const telemetryRows: MemoryV2ConceptRowRecord[] = [...telemetrySlugs].map(
     (slug) => ({
@@ -623,7 +640,7 @@ async function injectViaRouter(args: {
       simAssistantRerankBoost: 0,
       inRerankPool: false,
       spreadContribution: 0,
-      source: routerPicked.has(slug) ? "router" : "carry_over",
+      source: routerResult.sourceBySlug.get(slug) ?? "carry_over",
       status: "not_injected",
     }),
   );
@@ -712,7 +729,7 @@ interface RenderInjectionBlockResult {
  * the agent into wasted reads.
  */
 const INJECTION_HEADER =
-  "**CRITICAL:** These are page summaries. Read the page file if it looks relevant.";
+  'Use `file_read("memory/concepts/path/to/file.md")` to read the full pages for any of the injected memory summaries you want more information on.';
 
 /**
  * Render the inner content of the `<memory>` block for a list of slugs.
@@ -745,9 +762,9 @@ const INJECTION_HEADER =
  * fallback for pages predating the summary field). Skills sit after the
  * concept sections under `### Skills You Can Use`, and CLI subcommands sit
  * after the skills under `### CLI Commands You Can Use`. The leading
- * `**CRITICAL:**` line tells the agent how to read the block.
+ * instruction line tells the agent how to read the block.
  *
- *   **CRITICAL:** These are page summaries. Read the page file if it looks relevant.
+ *   Use `file_read("memory/concepts/path/to/file.md")` to read the full pages for any of the injected memory summaries you want more information on.
  *
  *   # memory/concepts/<concept-slug-1>.md
  *   <summary-1>