@vellumai/assistant 0.8.4 → 0.8.5

package/src/cli/commands/memory-v2-compare-render.ts

@@ -0,0 +1,115 @@
+/**
+ * Text rendering for `assistant memory v2 compare` — turns a
+ * `ComparisonReport` (the daemon route's response) into a human-readable
+ * summary and a per-turn breakdown.
+ *
+ * Lives CLI-side because formatting for the terminal is a presentation concern
+ * (mirroring the inline rendering in the `simulate` subcommand). It imports
+ * only the response *type* from the daemon — `cli/no-daemon-internals` permits
+ * type-only imports but forbids pulling in daemon runtime modules.
+ */
+
+import type { ComparisonReport } from "../../memory/v2/harness/runner.js";
+
+function sortedKs(report: ComparisonReport): number[] {
+  return [...report.ks].sort((a, b) => a - b);
+}
+
+/** Sum per-lane hit counts across every scored turn for one retriever. */
+function laneTotals(
+  report: ComparisonReport,
+  retrieverName: string,
+): Record<string, number> {
+  const totals: Record<string, number> = {};
+  for (const turn of report.perTurn) {
+    const ev = turn.byRetriever[retrieverName];
+    if (!ev) continue;
+    for (const [lane, count] of Object.entries(ev.hitsByLane)) {
+      totals[lane] = (totals[lane] ?? 0) + count;
+    }
+  }
+  return totals;
+}
+
+export function renderComparisonReport(report: ComparisonReport): string {
+  const ks = sortedKs(report);
+  const lines: string[] = [
+    "Memory Retrieval Comparison",
+    "===========================",
+    `Turns: considered ${report.turnsConsidered}, scored ${report.turnsScored}, skipped ${report.turnsSkipped}`,
+    `recall@k cutoffs: ${ks.join(", ")}`,
+    "",
+  ];
+
+  if (report.turnsScored === 0) {
+    lines.push(
+      "No turns scored — nothing to report.",
+      "(Every sampled turn was skipped — e.g. input reconstruction failed, or none matched the filter.)",
+    );
+    return lines.join("\n");
+  }
+
+  for (const retriever of report.retrievers) {
+    lines.push(`Retriever: ${retriever.name}`);
+    for (const k of ks) {
+      const value = retriever.aggregate.meanRecallAtK[k];
+      lines.push(
+        `  recall@${k}: ${value !== undefined ? value.toFixed(3) : "n/a"}`,
+      );
+    }
+    lines.push(
+      `  failures: ${(retriever.aggregate.failureRate * 100).toFixed(1)}%`,
+    );
+    if (retriever.aggregate.meanCostUsd !== undefined) {
+      lines.push(`  mean cost: $${retriever.aggregate.meanCostUsd.toFixed(4)}`);
+    }
+    const lanes = Object.entries(laneTotals(report, retriever.name)).sort(
+      (a, b) => b[1] - a[1],
+    );
+    const laneStr = lanes.map(([lane, count]) => `${lane}=${count}`).join(", ");
+    lines.push(`  hits by lane: ${laneStr.length > 0 ? laneStr : "(none)"}`);
+    lines.push("");
+  }
+
+  return lines.join("\n").trimEnd();
+}
+
+export function renderTurnTrace(
+  report: ComparisonReport,
+  conversationId: string,
+  turn: number,
+): string {
+  const entry = report.perTurn.find(
+    (t) => t.conversationId === conversationId && t.turn === turn,
+  );
+  if (!entry) {
+    return (
+      `Turn ${conversationId}:${turn} not found in the report. Only scored ` +
+      `turns appear here — skipped turns (e.g. failed input reconstruction) ` +
+      `are excluded. turnsScored=${report.turnsScored}, ` +
+      `turnsSkipped=${report.turnsSkipped}.`
+    );
+  }
+
+  const ks = sortedKs(report);
+  const header = `Turn ${conversationId}:${turn}`;
+  const lines: string[] = [header, "-".repeat(header.length)];
+
+  for (const [name, ev] of Object.entries(entry.byRetriever)) {
+    const recallStr = ks
+      .map((k) => `${k}:${(ev.recallAtK[k] ?? 0).toFixed(3)}`)
+      .join("  ");
+    lines.push(
+      `Retriever: ${name}`,
+      `  selected (${ev.selected.length}): ${ev.selected.join(", ") || "(none)"}`,
+      `  hits (${ev.hits.length}): ${ev.hits.join(", ") || "(none)"}`,
+      `  misses (${ev.misses.length}): ${ev.misses.join(", ") || "(none)"}`,
+      `  extras (${ev.extras.length}): ${ev.extras.join(", ") || "(none)"}`,
+      `  recall@k: ${recallStr}`,
+      "  (no descent trace — tier-based retriever)",
+      "",
+    );
+  }
+
+  return lines.join("\n").trimEnd();
+}

package/src/cli/commands/memory-v2.ts

@@ -19,6 +19,7 @@
 import type { Command } from "commander";
 
 import { cliIpcCall } from "../../ipc/cli-client.js";
+import type { ComparisonReport } from "../../memory/v2/harness/runner.js";
 import type {
   MemoryV2BackfillOp,
   MemoryV2BackfillResult,
@@ -29,6 +30,10 @@ import type {
 } from "../../runtime/routes/memory-v2-routes.js";
 import { registerCommand } from "../lib/register-command.js";
 import { log } from "../logger.js";
+import {
+  renderComparisonReport,
+  renderTurnTrace,
+} from "./memory-v2-compare-render.js";
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -469,7 +474,15 @@ Examples:
               "memory_v2_simulate_router",
               {
                 body: {
-                  query: opts.query,
+                  // The CLI flag is still named `--query` for backwards
+                  // compatibility. It becomes the just-arrived
+                  // `userMessage` of a single (empty assistant, user)
+                  // pair — i.e. a first-turn scenario. nowText uses
+                  // the server default (live NOW.md), preserving the
+                  // existing single-turn CLI semantics.
+                  recentTurnPairs: [
+                    { assistantMessage: "", userMessage: opts.query },
+                  ],
                   ...(configOverrides ? { configOverrides } : {}),
                 },
               },
@@ -560,6 +573,168 @@ Examples:
             }
           },
         );
+
+      // ── compare ─────────────────────────────────────────────────────────
+
+      v2.command("compare")
+        .description(
+          "Compare retrievers against the router's logged picks over a sample of real turns (read-only)",
+        )
+        .option(
+          "--limit <n>",
+          "How many historical turns to sample (default 20). Each re-runs the router = one LLM call.",
+        )
+        .option(
+          "--strategy <recent|random>",
+          "Sampling strategy over historical turns (default recent)",
+        )
+        .option(
+          "--k <list>",
+          "Comma-separated recall@k cutoffs (default 5,10,25,50)",
+        )
+        .option(
+          "--conversation <id>",
+          "Restrict to a conversation id (repeatable)",
+          (val: string, acc: string[]) => {
+            acc.push(val);
+            return acc;
+          },
+          [] as string[],
+        )
+        .option(
+          "--trace <conversationId:turn>",
+          "Print the per-retriever breakdown for one scored turn",
+        )
+        .option(
+          "--include-not-injected",
+          "Also count router picks cut by the injection cap as ground truth",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted report")
+        .addHelpText(
+          "after",
+          `
+Runs the comparison harness read-only: samples historical 'router'-mode turns
+from memory_v2_activation_logs, reconstructs each turn's inputs, re-runs each
+retriever, and scores selections against the logged picks (recall@k). NO writes.
+
+Cost: each scored turn re-runs the router (one LLM call), so --limit is the
+cost knob — start small. Today the only retriever is the router itself, so this
+is the harness self-test (router graded against its own logged picks); the gap
+from 1.0 is input-reconstruction drift (NOW.md / config moved since the turn).
+
+Examples:
+  $ assistant memory v2 compare --limit 20
+  $ assistant memory v2 compare --limit 50 --strategy random --k 5,10,25
+  $ assistant memory v2 compare --limit 20 --trace conv-abc:7
+  $ assistant memory v2 compare --limit 20 --json | jq '.retrievers[0].aggregate'`,
+        )
+        .action(
+          async (opts: {
+            limit?: string;
+            strategy?: string;
+            k?: string;
+            conversation?: string[];
+            trace?: string;
+            includeNotInjected?: boolean;
+            json?: boolean;
+          }) => {
+            let limit: number | undefined;
+            if (opts.limit !== undefined) {
+              const parsed = Number(opts.limit);
+              if (!Number.isInteger(parsed) || parsed < 1) {
+                log.error(
+                  `--limit must be a positive integer (got "${opts.limit}")`,
+                );
+                process.exitCode = 1;
+                return;
+              }
+              limit = parsed;
+            }
+
+            if (
+              opts.strategy !== undefined &&
+              opts.strategy !== "recent" &&
+              opts.strategy !== "random"
+            ) {
+              log.error(
+                `--strategy must be "recent" or "random" (got "${opts.strategy}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+
+            let ks: number[] | undefined;
+            if (opts.k !== undefined) {
+              ks = opts.k.split(",").map((s) => Number(s.trim()));
+              if (ks.some((k) => !Number.isInteger(k) || k < 1)) {
+                log.error(
+                  `--k must be a comma-separated list of positive integers (got "${opts.k}")`,
+                );
+                process.exitCode = 1;
+                return;
+              }
+            }
+
+            const conversationIds =
+              opts.conversation && opts.conversation.length > 0
+                ? opts.conversation
+                : undefined;
+
+            const result = await cliIpcCall<ComparisonReport>(
+              "memory_v2_compare_retrievers",
+              {
+                body: {
+                  ...(limit !== undefined ? { limit } : {}),
+                  ...(opts.strategy !== undefined
+                    ? { strategy: opts.strategy }
+                    : {}),
+                  ...(ks !== undefined ? { ks } : {}),
+                  ...(conversationIds !== undefined ? { conversationIds } : {}),
+                  ...(opts.includeNotInjected === true
+                    ? { includeNotInjected: true }
+                    : {}),
+                },
+              },
+            );
+
+            if (!result.ok) {
+              log.error(result.error ?? "Failed to compare retrievers");
+              process.exitCode = 1;
+              return;
+            }
+
+            const payload = result.result!;
+
+            if (opts.json === true) {
+              log.info(JSON.stringify(payload, null, 2));
+              return;
+            }
+
+            log.info(renderComparisonReport(payload));
+
+            if (opts.trace !== undefined) {
+              const sep = opts.trace.lastIndexOf(":");
+              if (sep <= 0 || sep === opts.trace.length - 1) {
+                log.error(
+                  `--trace must be "<conversationId>:<turn>" (got "${opts.trace}")`,
+                );
+                process.exitCode = 1;
+                return;
+              }
+              const conversationId = opts.trace.slice(0, sep);
+              const turn = Number(opts.trace.slice(sep + 1));
+              if (!Number.isInteger(turn)) {
+                log.error(
+                  `--trace turn must be an integer (got "${opts.trace.slice(sep + 1)}")`,
+                );
+                process.exitCode = 1;
+                return;
+              }
+              log.info("");
+              log.info(renderTurnTrace(payload, conversationId, turn));
+            }
+          },
+        );
     },
   });
 }

package/src/cli/commands/memory-v3-render.ts

@@ -0,0 +1,344 @@
+/**
+ * Text rendering for `assistant memory v3 validate` and `... tree`.
+ *
+ * Both functions are pure presentation: they take the daemon route's response
+ * shape and return a terminal-ready string. They live CLI-side (mirroring
+ * `memory-v2-compare-render.ts`) and import only the response *types* from the
+ * daemon route — `cli/no-daemon-internals` permits type-only imports but
+ * forbids pulling in daemon runtime modules.
+ */
+
+import type {
+  DescentPass,
+  LlmCallRecord,
+  MemoryV3SimulateResult,
+  MemoryV3TreeResult,
+  MemoryV3ValidateResult,
+} from "../../runtime/routes/memory-v3-routes.js";
+
+/**
+ * Render a {@link MemoryV3ValidateResult} into a counts summary plus the
+ * offending ids for each non-empty category. Categories with zero entries
+ * print `none` so a clean tree reads at a glance.
+ */
+export function renderValidationReport(report: MemoryV3ValidateResult): string {
+  const lines: string[] = [
+    "Memory v3 Tree Validation",
+    "=========================",
+    `Dangling child refs: ${report.danglingChildRefCount || "none"}`,
+  ];
+  for (const d of report.danglingChildRefs) {
+    lines.push(`  - ${d.node} → ${d.kind}:${d.ref}`);
+  }
+
+  lines.push(`Orphan pages: ${report.orphanPageCount || "none"}`);
+  for (const slug of report.orphanPages) {
+    lines.push(`  - ${slug}`);
+  }
+
+  lines.push(`Cycles: ${report.cycleCount || "none"}`);
+  for (const c of report.cycles) {
+    lines.push(`  - ${c.from} → ${c.to}`);
+  }
+
+  lines.push(`Stale index: ${report.staleIndexCount || "none"}`);
+  for (const s of report.staleIndex) {
+    lines.push(`  - ${s.node} (older than child ${s.child})`);
+  }
+
+  lines.push(
+    `Unknown edge targets: ${report.unknownEdgeTargetCount || "none"}`,
+  );
+  for (const e of report.unknownEdgeTargets) {
+    lines.push(`  - ${e.from} → ${e.to}`);
+  }
+
+  return lines.join("\n");
+}
+
+/**
+ * Whether the validation report has any defect in any category. The CLI uses
+ * this to set a non-zero exit code so `validate` is scriptable as a check.
+ */
+export function reportHasDefects(report: MemoryV3ValidateResult): boolean {
+  return (
+    report.danglingChildRefCount > 0 ||
+    report.orphanPageCount > 0 ||
+    report.cycleCount > 0 ||
+    report.staleIndexCount > 0 ||
+    report.unknownEdgeTargetCount > 0
+  );
+}
+
+/**
+ * Render a {@link MemoryV3TreeResult} as an indented tree rooted at `view.root`,
+ * descending `node:` children depth-first. A node reached more than once
+ * (shared DAG sub-node) is printed once with a `(↑ …)` re-entry marker rather
+ * than re-expanded, which also bounds output when the structure contains a
+ * cycle. `page:` children are printed as leaves under their parent node.
+ */
+export function renderTree(view: MemoryV3TreeResult): string {
+  const childrenById = new Map<string, MemoryV3TreeResult["nodes"][number]>();
+  for (const node of view.nodes) {
+    childrenById.set(node.id, node);
+  }
+
+  const lines: string[] = [];
+  const expanded = new Set<string>();
+
+  const walk = (nodeId: string, depth: number): void => {
+    const indent = "  ".repeat(depth);
+    const node = childrenById.get(nodeId);
+
+    if (!node) {
+      lines.push(`${indent}node:${nodeId} (missing)`);
+      return;
+    }
+
+    if (expanded.has(nodeId)) {
+      // Shared DAG sub-node (or a cycle's back-edge): print the reference but
+      // do not re-expand, so output stays finite and the re-entry is visible.
+      lines.push(`${indent}node:${nodeId} (↑ already shown)`);
+      return;
+    }
+    expanded.add(nodeId);
+    lines.push(`${indent}node:${nodeId}`);
+
+    for (const child of node.children) {
+      if (child.kind === "page") {
+        lines.push(`${"  ".repeat(depth + 1)}page:${child.ref}`);
+      } else {
+        walk(child.ref, depth + 1);
+      }
+    }
+  };
+
+  walk(view.root, 0);
+
+  if (lines.length === 0) {
+    lines.push("(empty tree)");
+  }
+
+  // Surface nodes that exist on disk but were never reached from the root —
+  // they would otherwise be invisible in a root-anchored print.
+  const unreached = view.nodes
+    .map((n) => n.id)
+    .filter((id) => !expanded.has(id))
+    .sort();
+  if (unreached.length > 0) {
+    lines.push("", `Unreachable nodes (${unreached.length}):`);
+    for (const id of unreached) {
+      lines.push(`  - node:${id}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+/** Canonical print order for the loop's provenance lanes. */
+const SIMULATE_LANE_ORDER = ["hot", "sparse", "dense", "tree", "edge"] as const;
+
+/** Render the effective lane toggles as a one-line `on` / restricted summary. */
+function renderEffectiveLanes(
+  lanes: MemoryV3SimulateResult["effectiveConfig"]["lanes"],
+): string {
+  const on = Object.entries(lanes)
+    .filter(([, enabled]) => enabled)
+    .map(([name]) => name);
+  const off = Object.entries(lanes)
+    .filter(([, enabled]) => !enabled)
+    .map(([name]) => name);
+  if (off.length === 0) return on.join(", ");
+  return `${on.join(", ")}  (off: ${off.join(", ")})`;
+}
+
+/** Render one pass's scout / tree / edge / gate breakdown into `lines`. */
+function renderPass(pass: DescentPass, lines: string[]): void {
+  lines.push(`Pass ${pass.passNumber}`);
+
+  if (pass.scouts && pass.scouts.length > 0) {
+    const summary = pass.scouts
+      .map((s) => `${s.lane}=${s.slugs.length}`)
+      .join("  ");
+    lines.push(`  scouts: ${summary}`);
+  }
+
+  if (pass.treeLevels && pass.treeLevels.length > 0) {
+    lines.push(`  tree: ${pass.treeLevels.length} level(s)`);
+    for (const level of pass.treeLevels) {
+      const node = level.node === "" ? "[root]" : level.node;
+      lines.push(
+        `    ${node}: considered ${level.considered.length}, descended ${level.descended.length}, skipped ${level.skipped.length}`,
+      );
+      if (level.descended.length > 0) {
+        lines.push(`      → ${level.descended.join(", ")}`);
+      }
+      if (level.reasoning.trim().length > 0) {
+        lines.push(`      reason: ${level.reasoning.trim()}`);
+      }
+    }
+  }
+
+  if (pass.edgeExpansions && pass.edgeExpansions.length > 0) {
+    const pulled = pass.edgeExpansions.reduce((n, e) => n + e.pulled.length, 0);
+    lines.push(
+      `  edges: ${pass.edgeExpansions.length} seed(s) expanded, ${pulled} pulled`,
+    );
+  }
+
+  if (pass.gate) {
+    lines.push(`  gate: ${pass.gate.decision}`);
+    for (const q of pass.gate.questions ?? []) {
+      lines.push(`    ? ${q}`);
+    }
+  }
+}
+
+/**
+ * Render a {@link MemoryV3SimulateResult} into a query echo, effective config,
+ * per-pass descent breakdown, and the final selection grouped by provenance
+ * lane (in fanout order). Mirrors the grouped layout of `memory v2 simulate`.
+ */
+export function renderSimulation(result: MemoryV3SimulateResult): string {
+  const lines: string[] = [
+    "Memory v3 Retrieval Simulation",
+    "==============================",
+    `Query: ${JSON.stringify(result.query)}`,
+    "",
+    "Config (effective):",
+    `  passCap: ${result.effectiveConfig.passCap}`,
+    `  lanes: ${renderEffectiveLanes(result.effectiveConfig.lanes)}`,
+    "",
+  ];
+
+  const passes = result.trace.passes;
+  lines.push(`Passes: ${passes.length || "none"}`);
+  for (const pass of passes) {
+    lines.push("");
+    renderPass(pass, lines);
+  }
+  lines.push("");
+
+  lines.push(`Selected: ${result.selectedSlugs.length} page(s)`);
+  const grouped = new Map<string, string[]>();
+  for (const slug of result.selectedSlugs) {
+    const lane = result.sourceBySlug[slug] ?? "unknown";
+    const bucket = grouped.get(lane) ?? [];
+    bucket.push(slug);
+    grouped.set(lane, bucket);
+  }
+  const laneRank = (lane: string): number => {
+    const i = SIMULATE_LANE_ORDER.indexOf(
+      lane as (typeof SIMULATE_LANE_ORDER)[number],
+    );
+    return i === -1 ? SIMULATE_LANE_ORDER.length : i;
+  };
+  const lanes = [...grouped.keys()].sort((a, b) => laneRank(a) - laneRank(b));
+  for (const lane of lanes) {
+    const slugs = grouped.get(lane)!;
+    lines.push(`  ${lane} (${slugs.length})`);
+    for (const slug of slugs) {
+      lines.push(`    - ${slug}`);
+    }
+  }
+
+  if (result.cost.ms !== undefined) {
+    lines.push("", `Cost: ${result.cost.ms} ms`);
+  }
+  if (result.failureReason) {
+    lines.push(`Failure: ${result.failureReason}`);
+  }
+
+  return lines.join("\n");
+}
+
+// ── LLM-call capture ────────────────────────────────────────────────────────
+
+type CapturedMessage = LlmCallRecord["request"]["messages"][number];
+type CapturedBlock = LlmCallRecord["response"]["content"][number];
+
+/** Concatenate the text blocks of a message (non-text blocks are ignored). */
+function messageText(message: CapturedMessage): string {
+  return message.content
+    .filter(
+      (b): b is Extract<CapturedBlock, { type: "text" }> => b.type === "text",
+    )
+    .map((b) => b.text)
+    .join("\n");
+}
+
+/** The forced-tool block from a response, if the model returned one. */
+function toolUseOf(
+  call: LlmCallRecord,
+): Extract<CapturedBlock, { type: "tool_use" }> | undefined {
+  return call.response.content.find(
+    (b): b is Extract<CapturedBlock, { type: "tool_use" }> =>
+      b.type === "tool_use",
+  );
+}
+
+function truncate(s: string, max: number): string {
+  const oneLine = s.replace(/\s+/g, " ").trim();
+  return oneLine.length > max ? `${oneLine.slice(0, max - 1)}…` : oneLine;
+}
+
+function indent(s: string, by = "    "): string {
+  return s
+    .split("\n")
+    .map((line) => by + line)
+    .join("\n");
+}
+
+/**
+ * Render the captured v3 LLM calls. Compact (default): one line per call —
+ * pass/lane/node, input size, round-trip ms, and the forced-tool summary. Full
+ * (`--show-llm`): the system prompt, every message, the tool names, and the
+ * tool_use input for each call. Grouped in call order (which is pass order).
+ */
+export function renderLlmCalls(
+  calls: readonly LlmCallRecord[],
+  opts: { full: boolean },
+): string {
+  if (calls.length === 0) return "LLM calls: none";
+
+  const lines: string[] = [`LLM calls (${calls.length}):`];
+  for (const call of calls) {
+    const label =
+      `pass${call.pass} · ${call.lane}` +
+      (call.node ? ` · node=${call.node}` : "");
+    const inputChars =
+      call.request.systemPrompt.length +
+      call.request.messages.reduce((n, m) => n + messageText(m).length, 0);
+    const tool = toolUseOf(call);
+    const toolSummary = tool
+      ? `${tool.name}(${truncate(JSON.stringify(tool.input), 80)})`
+      : `no tool_use (stop=${call.response.stopReason})`;
+
+    if (!opts.full) {
+      lines.push(
+        `  ${label} · in ${inputChars}c · ${call.ms}ms · ${truncate(toolSummary, 100)}`,
+      );
+      continue;
+    }
+
+    lines.push("", `── ${label}  (${call.callSite}, ${call.ms}ms) ──`);
+    lines.push("system:", indent(call.request.systemPrompt));
+    lines.push("messages:");
+    for (const message of call.request.messages) {
+      lines.push(
+        indent(`[${message.role}]`),
+        indent(messageText(message), "      "),
+      );
+    }
+    lines.push(`tools: ${call.request.tools.map((t) => t.name).join(", ")}`);
+    lines.push("output:");
+    lines.push(
+      indent(
+        tool
+          ? `${tool.name} ${JSON.stringify(tool.input, null, 2)}`
+          : `(no tool_use, stop=${call.response.stopReason})`,
+      ),
+    );
+  }
+  return lines.join("\n");
+}