@vellumai/assistant 0.8.3 → 0.8.5

package/src/cli/commands/__tests__/domain-status.test.ts

@@ -2,9 +2,9 @@
  * Tests for the domain status CLI subcommand (thin IPC wrapper).
  *
  * Validates:
- *   - status calls domain_status
+ *   - status calls domain_status then domain_verification_status
  *   - --json outputs structured response
- *   - no domain shows helpful message
+ *   - unknown subdomain shows helpful message
  *   - error responses are surfaced correctly
  */
 
@@ -99,15 +99,16 @@ async function runDomainCommand(...args: string[]) {
 // status
 // ---------------------------------------------------------------------------
 
-describe("assistant domain status", () => {
-  test("calls domain_status", async () => {
-    mockIpcCallFn = mock(() =>
-      Promise.resolve({
+function mockDomainStatusWithVerification() {
+  mockIpcCallFn = mock((method: string) => {
+    if (method === "domain_status") {
+      return Promise.resolve({
         ok: true,
         result: {
           results: [
             {
               id: "550e8400-e29b-41d4-a716-446655440000",
+              subdomain: "becky",
               domain: "becky.vellum.me",
               status: "active",
               verified: true,
@@ -115,38 +116,37 @@ describe("assistant domain status", () => {
             },
           ],
         },
-      }),
-    );
+      });
+    }
+    return Promise.resolve({
+      ok: true,
+      result: {
+        domain: "becky.vellum.me",
+        status: "verified",
+        message: "DNS records have been verified.",
+      },
+    });
+  }) as unknown as typeof mockIpcCallFn;
+}
 
-    await runDomainCommand("domain", "status");
+describe("assistant domain status", () => {
+  test("calls domain_status and domain_verification_status", async () => {
+    mockDomainStatusWithVerification();
 
-    expect(mockIpcCallFn).toHaveBeenCalledWith("domain_status");
+    await runDomainCommand("domain", "status", "becky");
+
+    expect(mockIpcCallFn).toHaveBeenCalledTimes(2);
     expect(process.exitCode).toBe(0);
   });
 
   test("--json outputs structured response", async () => {
-    mockIpcCallFn = mock(() =>
-      Promise.resolve({
-        ok: true,
-        result: {
-          results: [
-            {
-              id: "550e8400-e29b-41d4-a716-446655440000",
-              domain: "becky.vellum.me",
-              status: "active",
-              verified: true,
-              created_at: "2026-04-15T19:00:00Z",
-            },
-          ],
-        },
-      }),
-    );
+    mockDomainStatusWithVerification();
 
-    const output = await runDomainCommand("domain", "--json", "status");
+    const output = await runDomainCommand("domain", "--json", "status", "becky");
 
     const parsed = JSON.parse(output.trim());
-    expect(parsed.results).toHaveLength(1);
-    expect(parsed.results[0].domain).toBe("becky.vellum.me");
+    expect(parsed.domain.domain).toBe("becky.vellum.me");
+    expect(parsed.verification.status).toBe("verified");
     expect(process.exitCode).toBe(0);
   });
 
@@ -158,8 +158,8 @@ describe("assistant domain status", () => {
       }),
     );
 
-    await runDomainCommand("domain", "status");
-    expect(process.exitCode).toBe(0);
+    await runDomainCommand("domain", "status", "becky");
+    expect(process.exitCode).toBe(1);
   });
 
   test("IPC error with --json outputs error envelope", async () => {
@@ -171,7 +171,7 @@ describe("assistant domain status", () => {
       }),
     ) as unknown as typeof mockIpcCallFn;
 
-    const output = await runDomainCommand("domain", "--json", "status");
+    const output = await runDomainCommand("domain", "--json", "status", "becky");
 
     expect(process.exitCode).not.toBe(0);
     const parsed = JSON.parse(output.trim());
@@ -187,7 +187,7 @@ describe("assistant domain status", () => {
       }),
     ) as unknown as typeof mockIpcCallFn;
 
-    await runDomainCommand("domain", "status");
+    await runDomainCommand("domain", "status", "becky");
     expect(process.exitCode).not.toBe(0);
   });
 });

package/src/cli/commands/__tests__/inference-send.test.ts

@@ -6,8 +6,8 @@
  * rendering, argument validation, and the no-message guard. They run
  * entirely inside the CLI process and need no daemon stub.
  *
- * Follow-up opportunity: mock `../../../ipc/cli-client.js` with canned
- * responses to cover the deeper send-message paths against the IPC contract.
+ * The IPC client is mocked with canned responses so tests can assert the
+ * request contract without opening an assistant socket.
  */
 
 import {
@@ -23,6 +23,34 @@ import { Command } from "commander";
 // ---------------------------------------------------------------------------
 
 let mockStdinContent: string | null = null;
+let lastIpcCall: {
+  method: string;
+  params?: Record<string, unknown>;
+  options?: { timeoutMs?: number };
+} | null = null;
+let mockIpcResult: {
+  ok: boolean;
+  result?: unknown;
+  error?: string;
+} = {
+  ok: true,
+  result: {
+    response: "Hello from the model.",
+    model: "test-model",
+    usage: { inputTokens: 3, outputTokens: 4 },
+  },
+};
+
+mock.module("../../../ipc/cli-client.js", () => ({
+  cliIpcCall: async (
+    method: string,
+    params?: Record<string, unknown>,
+    options?: { timeoutMs?: number },
+  ) => {
+    lastIpcCall = { method, params, options };
+    return mockIpcResult;
+  },
+}));
 
 mock.module("../../../providers/provider-send-message.js", () => ({
   // The handler under test calls getConfiguredProvider before any of the
@@ -30,7 +58,10 @@ mock.module("../../../providers/provider-send-message.js", () => ({
   // loads cleanly even though no test actually drives a request.
   getConfiguredProvider: async () => null,
   extractAllText: () => "",
-  userMessage: (text: string) => ({ role: "user", content: [{ type: "text", text }] }),
+  userMessage: (text: string) => ({
+    role: "user",
+    content: [{ type: "text", text }],
+  }),
 }));
 
 mock.module("../../../config/loader.js", () => ({
@@ -44,8 +75,18 @@ mock.module("../../../config/loader.js", () => ({
 }));
 
 mock.module("../../../util/logger.js", () => ({
-  getLogger: () => ({ info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }),
-  getCliLogger: () => ({ info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }),
+  getLogger: () => ({
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+  }),
+  getCliLogger: () => ({
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+  }),
 }));
 
 mock.module("node:fs", () => ({
@@ -127,6 +168,15 @@ async function runCommand(
 
 beforeEach(() => {
   mockStdinContent = null;
+  lastIpcCall = null;
+  mockIpcResult = {
+    ok: true,
+    result: {
+      response: "Hello from the model.",
+      model: "test-model",
+      usage: { inputTokens: 3, outputTokens: 4 },
+    },
+  };
   process.exitCode = 0;
 });
 
@@ -142,6 +192,7 @@ describe("help text", () => {
     expect(stdout).toContain("--model");
     expect(stdout).toContain("--profile");
     expect(stdout).toContain("--max-tokens");
+    expect(stdout).toContain("--timeout-seconds");
     expect(stdout).toContain("--json");
     expect(stdout).toContain("[message...]");
   });
@@ -153,6 +204,7 @@ describe("help text", () => {
     expect(stdout).toContain("--model");
     expect(stdout).toContain("--profile");
     expect(stdout).toContain("--max-tokens");
+    expect(stdout).toContain("--timeout-seconds");
     expect(stdout).toContain("--json");
     expect(stdout).toContain("[message...]");
   });
@@ -225,3 +277,54 @@ describe("--max-tokens", () => {
     expect(parsed.error).toContain("Invalid --max-tokens");
   });
 });
+
+// ---------------------------------------------------------------------------
+// IPC timeout
+// ---------------------------------------------------------------------------
+
+describe("--timeout-seconds", () => {
+  test("uses a long default IPC timeout for inference calls", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "inference",
+      "send",
+      "--json",
+      "Hello",
+    ]);
+
+    expect(exitCode).toBe(0);
+    expect(JSON.parse(stdout).ok).toBe(true);
+    expect(lastIpcCall!.method).toBe("inference_send");
+    expect(lastIpcCall!.options!.timeoutMs).toBe(32 * 60 * 1000);
+  });
+
+  test("passes custom timeout to IPC call", async () => {
+    const { exitCode } = await runCommand([
+      "llm",
+      "send",
+      "--timeout-seconds",
+      "300",
+      "Hello",
+    ]);
+
+    expect(exitCode).toBe(0);
+    expect(lastIpcCall!.method).toBe("inference_send");
+    expect(lastIpcCall!.options!.timeoutMs).toBe(300_000);
+  });
+
+  test("errors on invalid timeout value", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "inference",
+      "send",
+      "--timeout-seconds",
+      "0",
+      "--json",
+      "Hello",
+    ]);
+
+    expect(exitCode).toBe(1);
+    expect(lastIpcCall).toBeNull();
+    const parsed = JSON.parse(stdout);
+    expect(parsed.ok).toBe(false);
+    expect(parsed.error).toContain("Invalid --timeout-seconds");
+  });
+});

package/src/cli/commands/__tests__/memory-v2-compare-render.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ComparisonReport } from "../../../memory/v2/harness/runner.js";
+import {
+  renderComparisonReport,
+  renderTurnTrace,
+} from "../memory-v2-compare-render.js";
+
+function sampleReport(): ComparisonReport {
+  return {
+    ks: [5, 10],
+    turnsConsidered: 3,
+    turnsScored: 1,
+    turnsSkipped: 2,
+    perTurn: [
+      {
+        conversationId: "c1",
+        turn: 7,
+        byRetriever: {
+          router: {
+            groundTruth: ["a", "b"],
+            selected: ["a", "x"],
+            hits: ["a"],
+            misses: ["b"],
+            extras: ["x"],
+            recallAtK: { 5: 0.5, 10: 0.5 },
+            hitsByLane: { tier1: 1 },
+            failureReason: null,
+          },
+        },
+      },
+    ],
+    retrievers: [
+      {
+        name: "router",
+        aggregate: {
+          turns: 1,
+          meanRecallAtK: { 5: 0.5, 10: 0.5 },
+          failureRate: 0,
+        },
+      },
+    ],
+  };
+}
+
+describe("memory v2 compare — renderComparisonReport", () => {
+  test("renders turn counts, recall@k, and per-lane attribution", () => {
+    const out = renderComparisonReport(sampleReport());
+    expect(out).toContain("considered 3, scored 1, skipped 2");
+    expect(out).toContain("Retriever: router");
+    expect(out).toContain("recall@5: 0.500");
+    expect(out).toContain("recall@10: 0.500");
+    expect(out).toContain("failures: 0.0%");
+    expect(out).toContain("hits by lane: tier1=1");
+  });
+
+  test("renders mean cost when present", () => {
+    const report = sampleReport();
+    report.retrievers[0]!.aggregate.meanCostUsd = 0.0123;
+    expect(renderComparisonReport(report)).toContain("mean cost: $0.0123");
+  });
+
+  test("handles a report with no scored turns", () => {
+    const out = renderComparisonReport({
+      ks: [5],
+      turnsConsidered: 4,
+      turnsScored: 0,
+      turnsSkipped: 4,
+      perTurn: [],
+      retrievers: [
+        {
+          name: "router",
+          aggregate: { turns: 0, meanRecallAtK: { 5: 0 }, failureRate: 0 },
+        },
+      ],
+    });
+    expect(out).toContain("No turns scored");
+  });
+});
+
+describe("memory v2 compare — renderTurnTrace", () => {
+  test("renders the per-retriever breakdown for a scored turn", () => {
+    const out = renderTurnTrace(sampleReport(), "c1", 7);
+    expect(out).toContain("Turn c1:7");
+    expect(out).toContain("Retriever: router");
+    expect(out).toContain("selected (2): a, x");
+    expect(out).toContain("hits (1): a");
+    expect(out).toContain("misses (1): b");
+    expect(out).toContain("extras (1): x");
+    expect(out).toContain("(no descent trace — tier-based retriever)");
+  });
+
+  test("explains when the requested turn was not scored", () => {
+    const out = renderTurnTrace(sampleReport(), "c1", 999);
+    expect(out).toContain("not found");
+    expect(out).toContain("turnsSkipped=2");
+  });
+});

package/src/cli/commands/__tests__/memory-v2.test.ts

@@ -2,9 +2,9 @@
  * Tests for the `assistant memory v2` CLI subgroup.
  *
  * Validates:
- *   - Subcommand registration (reembed, reembed-skills, activation, validate)
- *     under `memory v2`. The `memory` parent is created by the v2 registrar
- *     itself; v1 had its own subcommands but those were retired.
+ *   - Subcommand registration (reembed, reembed-skills, activation, validate,
+ *     ema, simulate) under `memory v2`. The `memory` parent is created by the
+ *     v2 registrar itself; v1 had its own subcommands but those were retired.
  *   - Each mutating subcommand maps to the right `memory_v2_backfill` op.
  *   - `validate` calls `memory_v2_validate` and pretty-prints the report.
  *   - `reembed-skills` calls `memory_v2_reembed_skills` synchronously.
@@ -143,8 +143,11 @@ describe("subcommand registration", () => {
     const subcommandNames = v2!.commands.map((c) => c.name()).sort();
     expect(subcommandNames).toEqual([
       "activation",
+      "compare",
+      "ema",
       "reembed",
       "reembed-skills",
+      "simulate",
       "validate",
     ]);
   });
@@ -158,6 +161,7 @@ describe("subcommand registration", () => {
     expect(help).toContain("reembed-skills");
     expect(help).toContain("activation");
     expect(help).toContain("validate");
+    expect(help).toContain("simulate");
     // Removed subcommands must not surface in help.
     expect(help).not.toContain("migrate");
     expect(help).not.toContain("explain");
@@ -307,9 +311,7 @@ describe("memory v2 validate", () => {
     expect(logOutput.some((line) => line.includes("Pages: 49"))).toBe(true);
     expect(logOutput.some((line) => line.includes("Edges: 166"))).toBe(true);
     expect(
-      logOutput.some((line) =>
-        line.includes("Missing edge endpoints: none"),
-      ),
+      logOutput.some((line) => line.includes("Missing edge endpoints: none")),
     ).toBe(true);
     expect(
       logOutput.some((line) => line.includes("Oversized pages: none")),
@@ -325,9 +327,7 @@ describe("memory v2 validate", () => {
       result: {
         pageCount: 10,
         edgeCount: 20,
-        missingEdgeEndpoints: [
-          { from: "people/alice", to: "people/missing" },
-        ],
+        missingEdgeEndpoints: [{ from: "people/alice", to: "people/missing" }],
         oversizedPages: [{ slug: "arcs/big-day", chars: 12345 }],
         parseFailures: [
           { slug: "people/broken", error: "missing frontmatter" },
@@ -342,9 +342,7 @@ describe("memory v2 validate", () => {
       true,
     );
     expect(logOutput.some((line) => line.includes("arcs/big-day"))).toBe(true);
-    expect(logOutput.some((line) => line.includes("people/broken"))).toBe(
-      true,
-    );
+    expect(logOutput.some((line) => line.includes("people/broken"))).toBe(true);
   });
 
   test("exits with code 1 on IPC failure", async () => {