@vellumai/assistant 0.8.4 → 0.8.5

package/src/cli/commands/__tests__/inference-send.test.ts

@@ -6,8 +6,8 @@
  * rendering, argument validation, and the no-message guard. They run
  * entirely inside the CLI process and need no daemon stub.
  *
- * Follow-up opportunity: mock `../../../ipc/cli-client.js` with canned
- * responses to cover the deeper send-message paths against the IPC contract.
+ * The IPC client is mocked with canned responses so tests can assert the
+ * request contract without opening an assistant socket.
  */
 
 import {
@@ -23,6 +23,34 @@ import { Command } from "commander";
 // ---------------------------------------------------------------------------
 
 let mockStdinContent: string | null = null;
+let lastIpcCall: {
+  method: string;
+  params?: Record<string, unknown>;
+  options?: { timeoutMs?: number };
+} | null = null;
+let mockIpcResult: {
+  ok: boolean;
+  result?: unknown;
+  error?: string;
+} = {
+  ok: true,
+  result: {
+    response: "Hello from the model.",
+    model: "test-model",
+    usage: { inputTokens: 3, outputTokens: 4 },
+  },
+};
+
+mock.module("../../../ipc/cli-client.js", () => ({
+  cliIpcCall: async (
+    method: string,
+    params?: Record<string, unknown>,
+    options?: { timeoutMs?: number },
+  ) => {
+    lastIpcCall = { method, params, options };
+    return mockIpcResult;
+  },
+}));
 
 mock.module("../../../providers/provider-send-message.js", () => ({
   // The handler under test calls getConfiguredProvider before any of the
@@ -30,7 +58,10 @@ mock.module("../../../providers/provider-send-message.js", () => ({
   // loads cleanly even though no test actually drives a request.
   getConfiguredProvider: async () => null,
   extractAllText: () => "",
-  userMessage: (text: string) => ({ role: "user", content: [{ type: "text", text }] }),
+  userMessage: (text: string) => ({
+    role: "user",
+    content: [{ type: "text", text }],
+  }),
 }));
 
 mock.module("../../../config/loader.js", () => ({
@@ -44,8 +75,18 @@ mock.module("../../../config/loader.js", () => ({
 }));
 
 mock.module("../../../util/logger.js", () => ({
-  getLogger: () => ({ info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }),
-  getCliLogger: () => ({ info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }),
+  getLogger: () => ({
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+  }),
+  getCliLogger: () => ({
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+  }),
 }));
 
 mock.module("node:fs", () => ({
@@ -127,6 +168,15 @@ async function runCommand(
 
 beforeEach(() => {
   mockStdinContent = null;
+  lastIpcCall = null;
+  mockIpcResult = {
+    ok: true,
+    result: {
+      response: "Hello from the model.",
+      model: "test-model",
+      usage: { inputTokens: 3, outputTokens: 4 },
+    },
+  };
   process.exitCode = 0;
 });
 
@@ -142,6 +192,7 @@ describe("help text", () => {
     expect(stdout).toContain("--model");
     expect(stdout).toContain("--profile");
     expect(stdout).toContain("--max-tokens");
+    expect(stdout).toContain("--timeout-seconds");
     expect(stdout).toContain("--json");
     expect(stdout).toContain("[message...]");
   });
@@ -153,6 +204,7 @@ describe("help text", () => {
     expect(stdout).toContain("--model");
     expect(stdout).toContain("--profile");
     expect(stdout).toContain("--max-tokens");
+    expect(stdout).toContain("--timeout-seconds");
     expect(stdout).toContain("--json");
     expect(stdout).toContain("[message...]");
   });
@@ -225,3 +277,54 @@ describe("--max-tokens", () => {
     expect(parsed.error).toContain("Invalid --max-tokens");
   });
 });
+
+// ---------------------------------------------------------------------------
+// IPC timeout
+// ---------------------------------------------------------------------------
+
+describe("--timeout-seconds", () => {
+  test("uses a long default IPC timeout for inference calls", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "inference",
+      "send",
+      "--json",
+      "Hello",
+    ]);
+
+    expect(exitCode).toBe(0);
+    expect(JSON.parse(stdout).ok).toBe(true);
+    expect(lastIpcCall!.method).toBe("inference_send");
+    expect(lastIpcCall!.options!.timeoutMs).toBe(32 * 60 * 1000);
+  });
+
+  test("passes custom timeout to IPC call", async () => {
+    const { exitCode } = await runCommand([
+      "llm",
+      "send",
+      "--timeout-seconds",
+      "300",
+      "Hello",
+    ]);
+
+    expect(exitCode).toBe(0);
+    expect(lastIpcCall!.method).toBe("inference_send");
+    expect(lastIpcCall!.options!.timeoutMs).toBe(300_000);
+  });
+
+  test("errors on invalid timeout value", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "inference",
+      "send",
+      "--timeout-seconds",
+      "0",
+      "--json",
+      "Hello",
+    ]);
+
+    expect(exitCode).toBe(1);
+    expect(lastIpcCall).toBeNull();
+    const parsed = JSON.parse(stdout);
+    expect(parsed.ok).toBe(false);
+    expect(parsed.error).toContain("Invalid --timeout-seconds");
+  });
+});

package/src/cli/commands/__tests__/memory-v2-compare-render.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ComparisonReport } from "../../../memory/v2/harness/runner.js";
+import {
+  renderComparisonReport,
+  renderTurnTrace,
+} from "../memory-v2-compare-render.js";
+
+function sampleReport(): ComparisonReport {
+  return {
+    ks: [5, 10],
+    turnsConsidered: 3,
+    turnsScored: 1,
+    turnsSkipped: 2,
+    perTurn: [
+      {
+        conversationId: "c1",
+        turn: 7,
+        byRetriever: {
+          router: {
+            groundTruth: ["a", "b"],
+            selected: ["a", "x"],
+            hits: ["a"],
+            misses: ["b"],
+            extras: ["x"],
+            recallAtK: { 5: 0.5, 10: 0.5 },
+            hitsByLane: { tier1: 1 },
+            failureReason: null,
+          },
+        },
+      },
+    ],
+    retrievers: [
+      {
+        name: "router",
+        aggregate: {
+          turns: 1,
+          meanRecallAtK: { 5: 0.5, 10: 0.5 },
+          failureRate: 0,
+        },
+      },
+    ],
+  };
+}
+
+describe("memory v2 compare — renderComparisonReport", () => {
+  test("renders turn counts, recall@k, and per-lane attribution", () => {
+    const out = renderComparisonReport(sampleReport());
+    expect(out).toContain("considered 3, scored 1, skipped 2");
+    expect(out).toContain("Retriever: router");
+    expect(out).toContain("recall@5: 0.500");
+    expect(out).toContain("recall@10: 0.500");
+    expect(out).toContain("failures: 0.0%");
+    expect(out).toContain("hits by lane: tier1=1");
+  });
+
+  test("renders mean cost when present", () => {
+    const report = sampleReport();
+    report.retrievers[0]!.aggregate.meanCostUsd = 0.0123;
+    expect(renderComparisonReport(report)).toContain("mean cost: $0.0123");
+  });
+
+  test("handles a report with no scored turns", () => {
+    const out = renderComparisonReport({
+      ks: [5],
+      turnsConsidered: 4,
+      turnsScored: 0,
+      turnsSkipped: 4,
+      perTurn: [],
+      retrievers: [
+        {
+          name: "router",
+          aggregate: { turns: 0, meanRecallAtK: { 5: 0 }, failureRate: 0 },
+        },
+      ],
+    });
+    expect(out).toContain("No turns scored");
+  });
+});
+
+describe("memory v2 compare — renderTurnTrace", () => {
+  test("renders the per-retriever breakdown for a scored turn", () => {
+    const out = renderTurnTrace(sampleReport(), "c1", 7);
+    expect(out).toContain("Turn c1:7");
+    expect(out).toContain("Retriever: router");
+    expect(out).toContain("selected (2): a, x");
+    expect(out).toContain("hits (1): a");
+    expect(out).toContain("misses (1): b");
+    expect(out).toContain("extras (1): x");
+    expect(out).toContain("(no descent trace — tier-based retriever)");
+  });
+
+  test("explains when the requested turn was not scored", () => {
+    const out = renderTurnTrace(sampleReport(), "c1", 999);
+    expect(out).toContain("not found");
+    expect(out).toContain("turnsSkipped=2");
+  });
+});

package/src/cli/commands/__tests__/memory-v2.test.ts

@@ -143,6 +143,7 @@ describe("subcommand registration", () => {
     const subcommandNames = v2!.commands.map((c) => c.name()).sort();
     expect(subcommandNames).toEqual([
       "activation",
+      "compare",
       "ema",
       "reembed",
       "reembed-skills",

package/src/cli/commands/__tests__/memory-v3-render.test.ts

@@ -0,0 +1,340 @@
+import { describe, expect, test } from "bun:test";
+
+import type {
+  LlmCallRecord,
+  MemoryV3SimulateResult,
+  MemoryV3TreeResult,
+  MemoryV3ValidateResult,
+} from "../../../runtime/routes/memory-v3-routes.js";
+import {
+  renderLlmCalls,
+  renderSimulation,
+  renderTree,
+  renderValidationReport,
+  reportHasDefects,
+} from "../memory-v3-render.js";
+
+function cleanReport(): MemoryV3ValidateResult {
+  return {
+    danglingChildRefs: [],
+    danglingChildRefCount: 0,
+    orphanPages: [],
+    orphanPageCount: 0,
+    cycles: [],
+    cycleCount: 0,
+    staleIndex: [],
+    staleIndexCount: 0,
+    unknownEdgeTargets: [],
+    unknownEdgeTargetCount: 0,
+  };
+}
+
+describe("memory v3 — renderValidationReport", () => {
+  test("renders 'none' for every empty category", () => {
+    const out = renderValidationReport(cleanReport());
+    expect(out).toContain("Memory v3 Tree Validation");
+    expect(out).toContain("Dangling child refs: none");
+    expect(out).toContain("Orphan pages: none");
+    expect(out).toContain("Cycles: none");
+    expect(out).toContain("Stale index: none");
+    expect(out).toContain("Unknown edge targets: none");
+  });
+
+  test("renders counts and offending ids for each defect category", () => {
+    const report: MemoryV3ValidateResult = {
+      danglingChildRefs: [{ node: "people", ref: "ghost", kind: "node" }],
+      danglingChildRefCount: 1,
+      orphanPages: ["stray-page"],
+      orphanPageCount: 1,
+      cycles: [{ from: "a", to: "b" }],
+      cycleCount: 1,
+      staleIndex: [
+        { node: "root", child: "people", nodeMtimeMs: 1, childMtimeMs: 2 },
+      ],
+      staleIndexCount: 1,
+      unknownEdgeTargets: [{ from: "p1", to: "missing" }],
+      unknownEdgeTargetCount: 1,
+    };
+    const out = renderValidationReport(report);
+    expect(out).toContain("Dangling child refs: 1");
+    expect(out).toContain("people → node:ghost");
+    expect(out).toContain("Orphan pages: 1");
+    expect(out).toContain("- stray-page");
+    expect(out).toContain("Cycles: 1");
+    expect(out).toContain("a → b");
+    expect(out).toContain("Stale index: 1");
+    expect(out).toContain("root (older than child people)");
+    expect(out).toContain("Unknown edge targets: 1");
+    expect(out).toContain("p1 → missing");
+  });
+});
+
+describe("memory v3 — reportHasDefects", () => {
+  test("false for a clean report", () => {
+    expect(reportHasDefects(cleanReport())).toBe(false);
+  });
+
+  test("true when any single category is non-empty", () => {
+    const report = cleanReport();
+    report.orphanPageCount = 1;
+    report.orphanPages = ["x"];
+    expect(reportHasDefects(report)).toBe(true);
+  });
+});
+
+describe("memory v3 — renderTree", () => {
+  test("renders an indented tree descending node and page children", () => {
+    const view: MemoryV3TreeResult = {
+      root: "_root",
+      nodes: [
+        {
+          id: "_root",
+          children: [
+            { kind: "node", ref: "people" },
+            { kind: "page", ref: "overview" },
+          ],
+        },
+        {
+          id: "people",
+          children: [{ kind: "page", ref: "alice" }],
+        },
+      ],
+    };
+    const out = renderTree(view);
+    expect(out).toBe(
+      ["node:_root", "  node:people", "    page:alice", "  page:overview"].join(
+        "\n",
+      ),
+    );
+  });
+
+  test("marks a shared DAG sub-node as a re-entry rather than re-expanding", () => {
+    const view: MemoryV3TreeResult = {
+      root: "_root",
+      nodes: [
+        {
+          id: "_root",
+          children: [
+            { kind: "node", ref: "a" },
+            { kind: "node", ref: "b" },
+          ],
+        },
+        { id: "a", children: [{ kind: "node", ref: "shared" }] },
+        { id: "b", children: [{ kind: "node", ref: "shared" }] },
+        { id: "shared", children: [{ kind: "page", ref: "leaf" }] },
+      ],
+    };
+    const out = renderTree(view);
+    // First reach under `a` expands; second reach under `b` is a marked re-entry.
+    expect(out).toContain("  node:a\n    node:shared\n      page:leaf");
+    expect(out).toContain("node:shared (↑ already shown)");
+    // The leaf page is expanded exactly once.
+    expect(out.match(/page:leaf/g)?.length).toBe(1);
+  });
+
+  test("bounds output on a cycle instead of looping forever", () => {
+    const view: MemoryV3TreeResult = {
+      root: "_root",
+      nodes: [
+        { id: "_root", children: [{ kind: "node", ref: "a" }] },
+        { id: "a", children: [{ kind: "node", ref: "_root" }] },
+      ],
+    };
+    const out = renderTree(view);
+    expect(out).toContain("node:_root (↑ already shown)");
+  });
+
+  test("flags a child ref whose target node is missing", () => {
+    const view: MemoryV3TreeResult = {
+      root: "_root",
+      nodes: [{ id: "_root", children: [{ kind: "node", ref: "ghost" }] }],
+    };
+    const out = renderTree(view);
+    expect(out).toContain("node:ghost (missing)");
+  });
+
+  test("lists nodes unreachable from the root", () => {
+    const view: MemoryV3TreeResult = {
+      root: "_root",
+      nodes: [
+        { id: "_root", children: [] },
+        { id: "floating", children: [] },
+      ],
+    };
+    const out = renderTree(view);
+    expect(out).toContain("Unreachable nodes (1):");
+    expect(out).toContain("- node:floating");
+  });
+});
+
+function simResult(): MemoryV3SimulateResult {
+  return {
+    query: "what should we ship next",
+    selectedSlugs: ["page-tree", "page-hot", "page-edge"],
+    sourceBySlug: {
+      "page-hot": "hot",
+      "page-tree": "tree",
+      "page-edge": "edge",
+    },
+    trace: {
+      passes: [
+        {
+          passNumber: 1,
+          scouts: [
+            { lane: "hot", slugs: ["page-hot"] },
+            { lane: "sparse", slugs: [] },
+            { lane: "dense", slugs: ["d1", "d2"] },
+          ],
+          treeLevels: [
+            {
+              node: "",
+              considered: ["people", "frames", "objects"],
+              descended: ["people", "frames"],
+              skipped: ["objects"],
+              reasoning: "query is about planning",
+            },
+          ],
+          edgeExpansions: [{ from: "page-tree", pulled: ["page-edge"] }],
+          gate: { decision: "more", questions: ["narrow to roadmap?"] },
+        },
+        {
+          passNumber: 2,
+          scouts: [{ lane: "hot", slugs: [] }],
+          gate: { decision: "ready" },
+        },
+      ],
+    },
+    cost: { ms: 1234 },
+    failureReason: null,
+    llmCalls: [],
+    effectiveConfig: {
+      passCap: 3,
+      lanes: { hot: true, sparse: true, dense: true, tree: true, edges: false },
+    },
+  };
+}
+
+describe("memory v3 — renderSimulation", () => {
+  test("renders query, effective config, per-pass trace, and grouped selection", () => {
+    const out = renderSimulation(simResult());
+    expect(out).toContain("Memory v3 Retrieval Simulation");
+    expect(out).toContain('Query: "what should we ship next"');
+    expect(out).toContain("passCap: 3");
+    // A disabled lane is surfaced in an `(off: …)` suffix.
+    expect(out).toContain("lanes: hot, sparse, dense, tree  (off: edges)");
+
+    expect(out).toContain("Passes: 2");
+    expect(out).toContain("Pass 1");
+    expect(out).toContain("scouts: hot=1  sparse=0  dense=2");
+    // The root tree level ("" node) prints as [root] with branch counts.
+    expect(out).toContain("[root]: considered 3, descended 2, skipped 1");
+    expect(out).toContain("→ people, frames");
+    expect(out).toContain("reason: query is about planning");
+    expect(out).toContain("edges: 1 seed(s) expanded, 1 pulled");
+    expect(out).toContain("gate: more");
+    expect(out).toContain("? narrow to roadmap?");
+    expect(out).toContain("gate: ready");
+
+    expect(out).toContain("Selected: 3 page(s)");
+    expect(out).toContain("Cost: 1234 ms");
+  });
+
+  test("groups selected slugs by provenance lane in fanout order", () => {
+    const out = renderSimulation(simResult());
+    const hotAt = out.indexOf("hot (1)");
+    const treeAt = out.indexOf("tree (1)");
+    const edgeAt = out.indexOf("edge (1)");
+    expect(hotAt).toBeGreaterThan(-1);
+    expect(treeAt).toBeGreaterThan(-1);
+    expect(edgeAt).toBeGreaterThan(-1);
+    // hot precedes tree precedes edge in SIMULATE_LANE_ORDER.
+    expect(hotAt).toBeLessThan(treeAt);
+    expect(treeAt).toBeLessThan(edgeAt);
+  });
+
+  test("renders all lanes inline when none are disabled", () => {
+    const result = simResult();
+    result.effectiveConfig.lanes.edges = true;
+    const out = renderSimulation(result);
+    expect(out).toContain("lanes: hot, sparse, dense, tree, edges");
+    expect(out).not.toContain("(off:");
+  });
+
+  test("surfaces a failure reason when the loop degraded", () => {
+    const result = simResult();
+    result.failureReason = "dense filter failed open";
+    const out = renderSimulation(result);
+    expect(out).toContain("Failure: dense filter failed open");
+  });
+});
+
+function llmCall(over: Partial<LlmCallRecord> = {}): LlmCallRecord {
+  return {
+    pass: 1,
+    lane: "gate",
+    callSite: "memoryV3Gate",
+    request: {
+      systemPrompt: "SYS-PROMPT",
+      messages: [
+        { role: "user", content: [{ type: "text", text: "USER-MSG" }] },
+      ],
+      tools: [
+        {
+          name: "decide_selection",
+          description: "decide",
+          input_schema: { type: "object", properties: {} },
+        },
+      ],
+    },
+    response: {
+      model: "stub-model",
+      stopReason: "tool_use",
+      usage: { inputTokens: 0, outputTokens: 0 },
+      content: [
+        {
+          type: "tool_use",
+          id: "tu-1",
+          name: "decide_selection",
+          input: { decision: "ready", selected_slugs: ["a"] },
+        },
+      ],
+    },
+    ms: 42,
+    ...over,
+  };
+}
+
+describe("memory v3 — renderLlmCalls", () => {
+  test("compact: one line per call with lane, node, and tool summary", () => {
+    const out = renderLlmCalls(
+      [
+        llmCall(),
+        llmCall({
+          lane: "descent",
+          callSite: "memoryV3Descent",
+          node: "people",
+        }),
+      ],
+      { full: false },
+    );
+    expect(out).toContain("LLM calls (2):");
+    expect(out).toContain("pass1 · gate");
+    expect(out).toContain("pass1 · descent · node=people");
+    expect(out).toContain("decide_selection");
+    expect(out).toContain("42ms");
+    // Compact mode must not dump the full system prompt.
+    expect(out).not.toContain("SYS-PROMPT");
+  });
+
+  test("full: includes system prompt, messages, tools, and tool_use input", () => {
+    const out = renderLlmCalls([llmCall()], { full: true });
+    expect(out).toContain("SYS-PROMPT");
+    expect(out).toContain("USER-MSG");
+    expect(out).toContain("decide_selection");
+    expect(out).toContain('"decision"');
+  });
+
+  test("renders 'none' when there are no calls", () => {
+    expect(renderLlmCalls([], { full: false })).toBe("LLM calls: none");
+  });
+});