@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/cli/commands/memory/__tests__/worker.test.ts

@@ -0,0 +1,302 @@
+/**
+ * Tests for the `assistant memory worker` CLI subgroup.
+ *
+ * Validates:
+ *   - Subcommand registration (start, stop, status) under `memory worker`.
+ *   - `status` reports running/not_running via PID-file liveness.
+ *   - `stop` sends SIGTERM to a live worker and errors when none is running.
+ *   - `start` refuses to spawn when a worker is already running, and reports
+ *     the PID once the spawned process writes its PID file.
+ */
+
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { Command } from "commander";
+
+// ---------------------------------------------------------------------------
+// Mock state
+// ---------------------------------------------------------------------------
+
+let tmpDir: string;
+let pidPath: string;
+let logOutput: string[] = [];
+
+/** Records (pid, signal) pairs passed to the mocked process.kill. */
+let killCalls: Array<{ pid: number; signal: string | number }> = [];
+
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+
+mock.module("../../../../util/platform.js", () => ({
+  getMemoryWorkerPidPath: () => pidPath,
+}));
+
+const capture = (...args: unknown[]) => {
+  logOutput.push(args.map(String).join(" "));
+};
+const fakeLogger = {
+  info: capture,
+  warn: capture,
+  error: capture,
+  debug: () => {},
+};
+mock.module("../../../../util/logger.js", () => ({
+  getLogger: () => fakeLogger,
+  getCliLogger: () => fakeLogger,
+}));
+
+// ---------------------------------------------------------------------------
+// Import module under test (after mocks)
+// ---------------------------------------------------------------------------
+
+const { registerMemoryWorkerCommand } = await import("../worker.js");
+
+// ---------------------------------------------------------------------------
+// Test helpers
+// ---------------------------------------------------------------------------
+
+function buildProgram(): Command {
+  const program = new Command();
+  program.exitOverride();
+  program.configureOutput({
+    writeErr: () => {},
+    writeOut: () => {},
+  });
+  const memory = program.command("memory");
+  registerMemoryWorkerCommand(memory);
+  return program;
+}
+
+async function runCommand(
+  args: string[],
+): Promise<{ stdout: string; exitCode: number }> {
+  const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+  const originalStderrWrite = process.stderr.write.bind(process.stderr);
+  const stdoutChunks: string[] = [];
+
+  process.stdout.write = ((chunk: unknown) => {
+    stdoutChunks.push(typeof chunk === "string" ? chunk : String(chunk));
+    return true;
+  }) as typeof process.stdout.write;
+
+  process.stderr.write = (() => true) as typeof process.stderr.write;
+
+  process.exitCode = 0;
+
+  try {
+    const program = buildProgram();
+    await program.parseAsync(["node", "assistant", ...args]);
+  } catch {
+    if (process.exitCode === 0) process.exitCode = 1;
+  } finally {
+    process.stdout.write = originalStdoutWrite;
+    process.stderr.write = originalStderrWrite;
+  }
+
+  const exitCode = process.exitCode ?? 0;
+  process.exitCode = 0;
+
+  return { exitCode, stdout: stdoutChunks.join("") };
+}
+
+/**
+ * Replace process.kill with a recording stub. `signal 0` is the liveness
+ * probe: it resolves for `livePids` and throws ESRCH otherwise. Other signals
+ * are recorded and no-oped so the test runner is never actually signalled.
+ */
+function stubProcessKill(livePids: Set<number>): () => void {
+  const original = process.kill.bind(process);
+  killCalls = [];
+  process.kill = ((pid: number, signal?: string | number) => {
+    const sig = signal ?? 0;
+    if (sig === 0) {
+      if (livePids.has(pid)) return true;
+      const err = new Error("kill ESRCH") as NodeJS.ErrnoException;
+      err.code = "ESRCH";
+      throw err;
+    }
+    killCalls.push({ pid, signal: sig });
+    return true;
+  }) as typeof process.kill;
+  return () => {
+    process.kill = original;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Setup
+// ---------------------------------------------------------------------------
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "memory-worker-test-"));
+  pidPath = join(tmpDir, "memory-worker.pid");
+  logOutput = [];
+  killCalls = [];
+  process.exitCode = 0;
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+// ---------------------------------------------------------------------------
+// Subcommand registration
+// ---------------------------------------------------------------------------
+
+describe("subcommand registration", () => {
+  test("registers worker under memory with start/stop/status", () => {
+    const program = buildProgram();
+    const memory = program.commands.find((c) => c.name() === "memory");
+    expect(memory).toBeDefined();
+    const worker = memory!.commands.find((c) => c.name() === "worker");
+    expect(worker).toBeDefined();
+    const subcommandNames = worker!.commands.map((c) => c.name()).sort();
+    expect(subcommandNames).toEqual(["start", "status", "stop"]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// status
+// ---------------------------------------------------------------------------
+
+describe("memory worker status", () => {
+  test("reports not_running when no PID file exists", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "memory",
+      "worker",
+      "status",
+      "--json",
+    ]);
+    expect(exitCode).toBe(0);
+    expect(JSON.parse(stdout)).toEqual({ status: "not_running" });
+  });
+
+  test("reports running when PID file points at a live process", async () => {
+    writeFileSync(pidPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "status",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toEqual({
+        status: "running",
+        pid: process.pid,
+      });
+    } finally {
+      restore();
+    }
+  });
+
+  test("treats a stale PID file as not_running and cleans it up", async () => {
+    writeFileSync(pidPath, "999999");
+    const restore = stubProcessKill(new Set());
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "status",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toEqual({ status: "not_running" });
+      expect(existsSync(pidPath)).toBe(false);
+    } finally {
+      restore();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// stop
+// ---------------------------------------------------------------------------
+
+describe("memory worker stop", () => {
+  test("errors with exit 1 when no worker is running", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "memory",
+      "worker",
+      "stop",
+      "--json",
+    ]);
+    expect(exitCode).toBe(1);
+    expect(JSON.parse(stdout)).toMatchObject({ ok: false });
+  });
+
+  test("sends SIGTERM to a running worker", async () => {
+    writeFileSync(pidPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "stop",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toEqual({ ok: true, pid: process.pid });
+      expect(killCalls).toContainEqual({
+        pid: process.pid,
+        signal: "SIGTERM",
+      });
+    } finally {
+      restore();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// start
+// ---------------------------------------------------------------------------
+
+describe("memory worker start", () => {
+  test("refuses to start when a worker is already running", async () => {
+    writeFileSync(pidPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "start",
+        "--json",
+      ]);
+      expect(exitCode).toBe(1);
+      expect(JSON.parse(stdout)).toMatchObject({
+        ok: false,
+        pid: process.pid,
+      });
+    } finally {
+      restore();
+    }
+  });
+
+  test("spawns the worker and reports the PID it writes", async () => {
+    const restore = stubProcessKill(new Set());
+    const originalSpawn = Bun.spawn;
+    // Simulate the spawned worker writing its PID file on startup.
+    (Bun as { spawn: typeof Bun.spawn }).spawn = (() => {
+      writeFileSync(pidPath, "424242");
+      return { unref: () => {}, pid: 424242 };
+    }) as unknown as typeof Bun.spawn;
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "start",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toMatchObject({ ok: true, pid: 424242 });
+    } finally {
+      (Bun as { spawn: typeof Bun.spawn }).spawn = originalSpawn;
+      restore();
+    }
+  });
+});

package/src/cli/commands/memory/index.ts

@@ -1,8 +1,10 @@
 import type { Command } from "commander";
 
 import { registerCommand } from "../../lib/register-command.js";
+import { registerMemoryRetrospectiveCommand } from "./memory-retrospective.js";
 import { registerMemoryV2Command } from "./memory-v2.js";
 import { registerMemoryV3Command } from "./memory-v3.js";
+import { registerMemoryWorkerCommand } from "./worker.js";
 
 export function registerMemoryCommand(program: Command): void {
   registerCommand(program, {
@@ -25,6 +27,8 @@ Examples:
 
       registerMemoryV2Command(memory);
       registerMemoryV3Command(memory);
+      registerMemoryRetrospectiveCommand(memory);
+      registerMemoryWorkerCommand(memory);
     },
   });
 }

package/src/cli/commands/memory/memory-retrospective.ts

@@ -0,0 +1,129 @@
+/**
+ * `assistant memory retrospective` CLI subgroup.
+ *
+ * Runs memory retrospectives directly in the CLI process — no IPC, no daemon
+ * required. Each subcommand imports the retrospective machinery and calls it
+ * against the workspace's on-disk SQLite database directly.
+ *
+ * Subcommands:
+ *
+ *   - `run <conversationId>` — run a fork-based retrospective on a conversation.
+ *   - `list` — list the last five retrospectives (TODO).
+ */
+
+import type { Command } from "commander";
+
+import { getConfig } from "../../../config/loader.js";
+import {
+  type MemoryRetrospectiveOutcome,
+  runForkBasedRetrospective,
+} from "../../../memory/memory-retrospective-job.js";
+import { registerCommand } from "../../lib/register-command.js";
+import { log } from "../../logger.js";
+import { shouldOutputJson, writeOutput } from "../../output.js";
+
+export function registerMemoryRetrospectiveCommand(memory: Command): void {
+  registerCommand(memory, {
+    name: "retrospective",
+    transport: "local",
+    description: "Run and inspect memory retrospectives (direct, no IPC)",
+    build: (retro) => {
+      retro.addHelpText(
+        "after",
+        `
+Runs memory retrospectives directly against the workspace database — the CLI
+process imports the retrospective machinery and calls it in-process, so no
+running daemon is required.
+
+Examples:
+  $ assistant memory retrospective run <conversationId>`,
+      );
+
+      // ── run ───────────────────────────────────────────────────────────────
+
+      retro
+        .command("run")
+        .description("Run a fork-based retrospective on a conversation")
+        .argument("<conversationId>", "Source conversation to retrospective")
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .addHelpText(
+          "after",
+          `
+Forks the source conversation through its latest message, persists a
+retrospective instruction, and wakes the fork so the agent reviews the new
+messages and calls \`remember\` on anything worth saving. Runs entirely in the
+CLI process — no IPC round-trip to the daemon.
+
+Examples:
+  $ assistant memory retrospective run abc123`,
+        )
+        .action(async (conversationId: string, opts: { json?: boolean }, cmd: Command) => {
+          const config = getConfig();
+          let outcome: MemoryRetrospectiveOutcome;
+          try {
+            outcome = await runForkBasedRetrospective(conversationId, config);
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            log.error(
+              { err, conversationId },
+              "memory-retrospective: run threw",
+            );
+            if (opts.json === true) {
+              writeOutput(cmd, { kind: "error", error: msg });
+            } else {
+              log.error(msg);
+            }
+            process.exitCode = 1;
+            return;
+          }
+
+          if (shouldOutputJson(cmd)) {
+            writeOutput(cmd, outcome);
+            return;
+          }
+
+          renderOutcome(outcome);
+        });
+    },
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Human-readable rendering
+// ---------------------------------------------------------------------------
+
+function renderOutcome(outcome: MemoryRetrospectiveOutcome): void {
+  switch (outcome.kind) {
+    case "disabled":
+      log.info("Retrospective is disabled for this workspace.");
+      break;
+    case "no_new_messages":
+      log.info("No new messages to review since the last retrospective.");
+      break;
+    case "source_processing":
+      log.info(
+        "Source conversation is mid-turn; skipping (will retry next trigger).",
+      );
+      break;
+    case "wake_failed":
+      log.error(
+        `Wake failed${outcome.reason ? `: ${outcome.reason}` : ""}` +
+          (outcome.conversationId
+            ? ` (fork: ${outcome.conversationId})`
+            : ""),
+      );
+      process.exitCode = 1;
+      break;
+    case "invoked":
+      log.info(
+        `Retrospective invoked.\n` +
+          `  fork conversation: ${outcome.backgroundConversationId}\n` +
+          `  cutoff message:    ${outcome.cutoffMessageId}\n` +
+          `  new messages:      ${outcome.newMessageCount}` +
+          (outcome.followUpJobIds.length > 0
+            ? `\n  follow-up jobs:    ${outcome.followUpJobIds.join(", ")}`
+            : ""),
+      );
+      break;
+  }
+}

package/src/cli/commands/memory/memory-v3.ts

@@ -15,10 +15,15 @@
  *     starts empty and the incremental maintain pass only re-embeds deltas.
  */
 
+import { readFileSync, writeFileSync } from "node:fs";
+
 import type { Command } from "commander";
 
 import { cliIpcCall } from "../../../ipc/cli-client.js";
-import type { MemoryEvalRunResult } from "../../../runtime/routes/memory-eval-routes.js";
+import type {
+  MemoryEvalRunResult,
+  MemoryEvalTallyResult,
+} from "../../../runtime/routes/memory-eval-routes.js";
 import type {
   MemoryV3BackfillSectionsResult,
   MemoryV3RebuildIndexResult,
@@ -42,6 +47,56 @@ const BACKFILL_IPC_TIMEOUT_MS = 30 * 60 * 1000;
  */
 const EVAL_IPC_TIMEOUT_MS = 30 * 60 * 1000;
 
+/** Commander accumulator for a repeatable `--exclude-conversation <id>` flag. */
+function collectRepeatable(value: string, acc: string[]): string[] {
+  return [...acc, value];
+}
+
+/**
+ * Read the `turn` ids from a prior run's `key.json` or `packets.json` (both are
+ * arrays of objects carrying a `turn` field). Used to pin `--turns-file` so a
+ * re-judge measures the exact same turn set.
+ */
+function readTurnIdsFromFile(path: string): string[] {
+  const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
+  if (!Array.isArray(parsed)) {
+    throw new Error(
+      `--turns-file ${path} must be a JSON array (a key.json or packets.json)`,
+    );
+  }
+  const ids = parsed
+    .map((e) => (e as { turn?: unknown }).turn)
+    .filter((t): t is string => typeof t === "string");
+  if (ids.length === 0) {
+    throw new Error(`--turns-file ${path} contained no turn ids`);
+  }
+  return ids;
+}
+
+/** Read and parse a JSON file, asserting it is an array. */
+function readJsonArray<T>(path: string, label: string): T[] {
+  const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
+  if (!Array.isArray(parsed)) {
+    throw new Error(`--${label} ${path} must be a JSON array`);
+  }
+  return parsed as T[];
+}
+
+/** Render an eval-tally result as a human-readable summary block. */
+function formatTally(r: MemoryEvalTallyResult): string {
+  const lines = [
+    `Eval gate: ${r.gate.toUpperCase()} (${r.verdict})`,
+    `  turns judged:   ${r.turns} (panel ${r.panel.min}-${r.panel.max} votes/turn, mean ${r.panel.mean.toFixed(1)})`,
+    `  snapshot (v2):  ${r.snapshotWins} wins, mean score ${r.meanSnapshot.toFixed(2)}`,
+    `  staging (wiki): ${r.stagingWins} wins, mean score ${r.meanStaging.toFixed(2)}`,
+    `  ties:           ${r.ties}`,
+    `  sign-test p:    ${r.signTestP.toFixed(3)} over ${r.decided} decided turns`,
+    `  confident:      ${r.confident ? "yes" : "no"}`,
+  ];
+  for (const note of r.notes) lines.push(`  note: ${note}`);
+  return lines.join("\n");
+}
+
 export function registerMemoryV3Command(memory: Command): void {
   registerCommand(memory, {
     name: "v3",
@@ -162,6 +217,16 @@ Examples:
           "--no-dense",
           "Needle-only: skip section embedding (fast, cheaper, lower fidelity)",
         )
+        .option(
+          "--turns-file <path>",
+          "Pin the exact turns from a prior key.json/packets.json (reproducible re-judge); overrides --turns",
+        )
+        .option(
+          "--exclude-conversation <id>",
+          "Conversation id to omit from mining (repeatable; e.g. the migration's own chat)",
+          collectRepeatable,
+          [],
+        )
         .option("--json", "Emit raw JSON instead of a formatted summary")
         .addHelpText(
           "after",
@@ -173,9 +238,17 @@ is touched. With the dense lane on (default) it embeds every section of both
 corpora, which can take a while on a large corpus; use --no-dense for a fast
 lexical-only pass.
 
+To iterate on the staged wiki reproducibly, mine the turns ONCE and pin them on
+every re-run with --turns-file (pointing at the first run's key.json), so the
+comparison stays fixed while only the staged corpus changes. Re-runs that
+re-mine drift onto a different turn set and are not comparable. Likewise, do not
+compare a --no-dense run against a dense one, and check eval-meta.json's
+embedding identity is the same across runs.
+
 Examples:
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval
-  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns 50 --no-dense`,
+  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns-file .mv3/eval/key.json
+  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --exclude-conversation <migration-conv-id>`,
         )
         .action(
           async (opts: {
@@ -186,8 +259,20 @@ Examples:
             k: string;
             seed: string;
             dense: boolean;
+            turnsFile?: string;
+            excludeConversation: string[];
             json?: boolean;
           }) => {
+            let turnIds: string[] | undefined;
+            if (opts.turnsFile !== undefined) {
+              try {
+                turnIds = readTurnIdsFromFile(opts.turnsFile);
+              } catch (err) {
+                log.error(err instanceof Error ? err.message : String(err));
+                process.exitCode = 1;
+                return;
+              }
+            }
             const result = await cliIpcCall<MemoryEvalRunResult>(
               "memory_eval_run",
               {
@@ -199,6 +284,10 @@ Examples:
                   k: Number(opts.k),
                   seed: Number(opts.seed),
                   dense: opts.dense,
+                  ...(turnIds ? { turnIds } : {}),
+                  ...(opts.excludeConversation.length > 0
+                    ? { excludeConversationIds: opts.excludeConversation }
+                    : {}),
                 },
               },
               { timeoutMs: EVAL_IPC_TIMEOUT_MS },
@@ -213,11 +302,94 @@ Examples:
               log.info(JSON.stringify(payload, null, 2));
               return;
             }
+            const emb = payload.embedding;
+            const embStr = payload.dense
+              ? `${emb.provider}/${emb.model}/${emb.dims ?? "?"}d`
+              : "needle-only (no dense)";
             log.info(
-              `Wrote ${payload.packetsWritten} packets from ${payload.turnsMined} turns ` +
+              `Wrote ${payload.packetsWritten} packets from ${payload.turnsMined}/${payload.turnsRequested} turns ` +
                 `(snapshot ${payload.snapshotPages} pages vs staged ${payload.stagingPages} pages, ` +
-                `dense=${payload.dense}).\n  packets: ${payload.packetsPath}\n  key:     ${payload.keyPath}`,
+                `dense=${payload.dense}, embedding=${embStr}).\n` +
+                `  packets: ${payload.packetsPath}\n  key:     ${payload.keyPath}\n  meta:    ${payload.metaPath}\n` +
+                `  Re-judge reproducibly with: --turns-file ${payload.keyPath}`,
+            );
+            if (payload.turnsMined < payload.turnsRequested) {
+              log.warn(
+                `Only ${payload.turnsMined} of ${payload.turnsRequested} requested turns were mined ` +
+                  `(pinned turns may have been deleted, or there are too few recent user turns).`,
+              );
+            }
+          },
+        );
+
+      v3.command("eval-tally")
+        .description(
+          "Unblind + tally blind-judge verdicts against key.json with a noise-aware win/tie/loss verdict",
+        )
+        .requiredOption(
+          "--verdicts <path>",
+          "JSON file: array of { turn, winner, scoreA, scoreB } (one or more per turn for a panel)",
+        )
+        .requiredOption(
+          "--key <path>",
+          "key.json from `eval` — the per-turn A/B → snapshot/staging unblinding map",
+        )
+        .option(
+          "--alpha <p>",
+          "Significance threshold for the sign test (the wiki only FAILS on a significant snapshot lead)",
+          "0.05",
+        )
+        .option("--out <path>", "Also write the full tally JSON to this path")
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .addHelpText(
+          "after",
+          `
+Joins the blind-judge verdicts to the unblinding key (A/B is shuffled PER TURN,
+so the winner must be mapped turn-by-turn — a global A-vs-B count is wrong) and
+applies a two-sided sign test: the wiki only FAILS when the snapshot's win lead
+is statistically significant. A within-noise difference is a tie, which passes
+the win-or-tie gate. Pass a judge PANEL (multiple verdicts per turn, e.g. from
+re-judging under several seeds) to control single-vote noise.
+
+Example:
+  $ assistant memory v3 eval-tally --verdicts .mv3/eval/verdicts.json --key .mv3/eval/key.json`,
+        )
+        .action(
+          async (opts: {
+            verdicts: string;
+            key: string;
+            alpha: string;
+            out?: string;
+            json?: boolean;
+          }) => {
+            let verdicts: unknown[];
+            let key: unknown[];
+            try {
+              verdicts = readJsonArray<unknown>(opts.verdicts, "verdicts");
+              key = readJsonArray<unknown>(opts.key, "key");
+            } catch (err) {
+              log.error(err instanceof Error ? err.message : String(err));
+              process.exitCode = 1;
+              return;
+            }
+            const result = await cliIpcCall<MemoryEvalTallyResult>(
+              "memory_eval_tally",
+              { body: { verdicts, key, alpha: Number(opts.alpha) } },
             );
+            if (!result.ok) {
+              log.error(result.error ?? "Failed to tally verdicts");
+              process.exitCode = 1;
+              return;
+            }
+            const payload = result.result!;
+            if (opts.out !== undefined) {
+              writeFileSync(opts.out, JSON.stringify(payload, null, 2));
+            }
+            if (opts.json === true) {
+              log.info(JSON.stringify(payload, null, 2));
+              return;
+            }
+            log.info(formatTally(payload));
           },
         );
     },