@vellumai/assistant 0.10.2 → 0.10.3-dev.202606252046.9075fd5

package/src/cli/commands/memory/__tests__/worker.test.ts

@@ -0,0 +1,432 @@
+/**
+ * Tests for the `assistant memory worker` CLI subgroup.
+ *
+ * Validates:
+ *   - Subcommand registration (start, stop, status) under `memory worker`.
+ *   - `status` reports the worker process state, `memory.worker.enabled`, and
+ *     the synchronous in-process runner via PID/marker-file liveness.
+ *   - `stop` sends SIGTERM to a live worker and disables the config flag, and
+ *     still disables the flag (success) when no worker is running.
+ *   - `start` spawns/reuses the worker process and enables the config flag.
+ */
+
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { Command } from "commander";
+
+// ---------------------------------------------------------------------------
+// Mock state
+// ---------------------------------------------------------------------------
+
+let tmpDir: string;
+let pidPath: string;
+let markerPath: string;
+let logOutput: string[] = [];
+
+/** In-memory stand-in for the on-disk raw config the CLI reads/writes. */
+let rawConfig: Record<string, unknown> = {};
+
+/** Records (pid, signal) pairs passed to the mocked process.kill. */
+let killCalls: Array<{ pid: number; signal: string | number }> = [];
+
+function workerEnabledFromRaw(): boolean {
+  const memory = rawConfig.memory as { worker?: { enabled?: unknown } };
+  return memory?.worker?.enabled === true;
+}
+
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+
+mock.module("../../../../util/platform.js", () => ({
+  getMemoryWorkerPidPath: () => pidPath,
+  getMemorySyncRunnerMarkerPath: () => markerPath,
+}));
+
+mock.module("../../../../config/loader.js", () => ({
+  loadRawConfig: () => structuredClone(rawConfig),
+  saveRawConfig: (config: Record<string, unknown>) => {
+    rawConfig = structuredClone(config);
+  },
+  getConfigReadOnly: () => ({
+    memory: { worker: { enabled: workerEnabledFromRaw() } },
+  }),
+  setNestedValue: (
+    obj: Record<string, unknown>,
+    path: string,
+    value: unknown,
+  ) => {
+    const keys = path.split(".");
+    let cur = obj;
+    for (let i = 0; i < keys.length - 1; i++) {
+      const k = keys[i];
+      if (cur[k] == null || typeof cur[k] !== "object") cur[k] = {};
+      cur = cur[k] as Record<string, unknown>;
+    }
+    cur[keys[keys.length - 1]] = value;
+  },
+}));
+
+const capture = (...args: unknown[]) => {
+  logOutput.push(args.map(String).join(" "));
+};
+const fakeLogger = {
+  info: capture,
+  warn: capture,
+  error: capture,
+  debug: () => {},
+};
+mock.module("../../../../util/logger.js", () => ({
+  getLogger: () => fakeLogger,
+  getCliLogger: () => fakeLogger,
+  getCurrentLogFilePath: () => `${tmpDir}/assistant-test-mock.log`,
+}));
+
+// ---------------------------------------------------------------------------
+// Import module under test (after mocks)
+// ---------------------------------------------------------------------------
+
+const { registerMemoryWorkerCommand } = await import("../worker.js");
+
+// ---------------------------------------------------------------------------
+// Test helpers
+// ---------------------------------------------------------------------------
+
+function buildProgram(): Command {
+  const program = new Command();
+  program.exitOverride();
+  program.configureOutput({
+    writeErr: () => {},
+    writeOut: () => {},
+  });
+  const memory = program.command("memory");
+  registerMemoryWorkerCommand(memory);
+  return program;
+}
+
+async function runCommand(
+  args: string[],
+): Promise<{ stdout: string; exitCode: number }> {
+  const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+  const originalStderrWrite = process.stderr.write.bind(process.stderr);
+  const stdoutChunks: string[] = [];
+
+  process.stdout.write = ((chunk: unknown) => {
+    stdoutChunks.push(typeof chunk === "string" ? chunk : String(chunk));
+    return true;
+  }) as typeof process.stdout.write;
+
+  process.stderr.write = (() => true) as typeof process.stderr.write;
+
+  process.exitCode = 0;
+
+  try {
+    const program = buildProgram();
+    await program.parseAsync(["node", "assistant", ...args]);
+  } catch {
+    if (process.exitCode === 0) process.exitCode = 1;
+  } finally {
+    process.stdout.write = originalStdoutWrite;
+    process.stderr.write = originalStderrWrite;
+  }
+
+  const exitCode = process.exitCode ?? 0;
+  process.exitCode = 0;
+
+  return { exitCode, stdout: stdoutChunks.join("") };
+}
+
+/**
+ * Replace process.kill with a recording stub. `signal 0` is the liveness
+ * probe: it resolves for `livePids` and throws ESRCH otherwise. Other signals
+ * are recorded and no-oped so the test runner is never actually signalled.
+ */
+function stubProcessKill(livePids: Set<number>): () => void {
+  const original = process.kill.bind(process);
+  killCalls = [];
+  process.kill = ((pid: number, signal?: string | number) => {
+    const sig = signal ?? 0;
+    if (sig === 0) {
+      if (livePids.has(pid)) return true;
+      const err = new Error("kill ESRCH") as NodeJS.ErrnoException;
+      err.code = "ESRCH";
+      throw err;
+    }
+    killCalls.push({ pid, signal: sig });
+    return true;
+  }) as typeof process.kill;
+  return () => {
+    process.kill = original;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Setup
+// ---------------------------------------------------------------------------
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "memory-worker-test-"));
+  pidPath = join(tmpDir, "memory-worker.pid");
+  markerPath = join(tmpDir, "memory-sync-runner.pid");
+  logOutput = [];
+  killCalls = [];
+  rawConfig = {};
+  process.exitCode = 0;
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+// ---------------------------------------------------------------------------
+// Subcommand registration
+// ---------------------------------------------------------------------------
+
+describe("subcommand registration", () => {
+  test("registers worker under memory with start/stop/status", () => {
+    const program = buildProgram();
+    const memory = program.commands.find((c) => c.name() === "memory");
+    expect(memory).toBeDefined();
+    const worker = memory!.commands.find((c) => c.name() === "worker");
+    expect(worker).toBeDefined();
+    const subcommandNames = worker!.commands.map((c) => c.name()).sort();
+    expect(subcommandNames).toEqual(["start", "status", "stop"]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// status
+// ---------------------------------------------------------------------------
+
+describe("memory worker status", () => {
+  test("reports not_running with workerEnabled and syncRunner when nothing is running", async () => {
+    const { exitCode, stdout } = await runCommand([
+      "memory",
+      "worker",
+      "status",
+      "--json",
+    ]);
+    expect(exitCode).toBe(0);
+    expect(JSON.parse(stdout)).toEqual({
+      status: "not_running",
+      workerEnabled: false,
+      syncRunner: { status: "not_running" },
+    });
+  });
+
+  test("reports running when PID file points at a live process", async () => {
+    writeFileSync(pidPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "status",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toEqual({
+        status: "running",
+        pid: process.pid,
+        workerEnabled: false,
+        syncRunner: { status: "not_running" },
+      });
+    } finally {
+      restore();
+    }
+  });
+
+  test("reflects memory.worker.enabled from config", async () => {
+    rawConfig = { memory: { worker: { enabled: true } } };
+    const restore = stubProcessKill(new Set());
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "status",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toMatchObject({ workerEnabled: true });
+    } finally {
+      restore();
+    }
+  });
+
+  test("reports the synchronous runner as running when its marker is live", async () => {
+    writeFileSync(markerPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "status",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toMatchObject({
+        status: "not_running",
+        syncRunner: { status: "running", pid: process.pid },
+      });
+    } finally {
+      restore();
+    }
+  });
+
+  test("treats a stale PID file as not_running and cleans it up", async () => {
+    writeFileSync(pidPath, "999999");
+    const restore = stubProcessKill(new Set());
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "status",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toEqual({
+        status: "not_running",
+        workerEnabled: false,
+        syncRunner: { status: "not_running" },
+      });
+      expect(existsSync(pidPath)).toBe(false);
+    } finally {
+      restore();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// stop
+// ---------------------------------------------------------------------------
+
+describe("memory worker stop", () => {
+  test("disables the config flag and succeeds when no worker is running", async () => {
+    rawConfig = { memory: { worker: { enabled: true } } };
+    const { exitCode, stdout } = await runCommand([
+      "memory",
+      "worker",
+      "stop",
+      "--json",
+    ]);
+    expect(exitCode).toBe(0);
+    expect(JSON.parse(stdout)).toMatchObject({
+      ok: true,
+      workerWasRunning: false,
+      workerEnabled: false,
+    });
+    expect(workerEnabledFromRaw()).toBe(false);
+  });
+
+  test("sends SIGTERM to a running worker and disables the flag", async () => {
+    rawConfig = { memory: { worker: { enabled: true } } };
+    writeFileSync(pidPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "stop",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toEqual({
+        ok: true,
+        pid: process.pid,
+        workerEnabled: false,
+      });
+      expect(killCalls).toContainEqual({
+        pid: process.pid,
+        signal: "SIGTERM",
+      });
+      expect(workerEnabledFromRaw()).toBe(false);
+    } finally {
+      restore();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// start
+// ---------------------------------------------------------------------------
+
+describe("memory worker start", () => {
+  test("reuses an already-running worker and enables the flag", async () => {
+    writeFileSync(pidPath, String(process.pid));
+    const restore = stubProcessKill(new Set([process.pid]));
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "start",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toMatchObject({
+        ok: true,
+        pid: process.pid,
+        alreadyRunning: true,
+        workerEnabled: true,
+      });
+      expect(workerEnabledFromRaw()).toBe(true);
+    } finally {
+      restore();
+    }
+  });
+
+  test("spawns the worker, reports its PID, and enables the flag", async () => {
+    const restore = stubProcessKill(new Set());
+    const originalSpawn = Bun.spawn;
+    // Simulate the spawned worker writing its PID file on startup.
+    (Bun as { spawn: typeof Bun.spawn }).spawn = (() => {
+      writeFileSync(pidPath, "424242");
+      return { unref: () => {}, pid: 424242 };
+    }) as unknown as typeof Bun.spawn;
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "start",
+        "--json",
+      ]);
+      expect(exitCode).toBe(0);
+      expect(JSON.parse(stdout)).toMatchObject({
+        ok: true,
+        pid: 424242,
+        workerEnabled: true,
+      });
+      expect(workerEnabledFromRaw()).toBe(true);
+    } finally {
+      (Bun as { spawn: typeof Bun.spawn }).spawn = originalSpawn;
+      restore();
+    }
+  });
+
+  test("leaves the config flag untouched when the spawn fails", async () => {
+    const restore = stubProcessKill(new Set());
+    const originalSpawn = Bun.spawn;
+    // Spawn returns but never writes a PID file → spawnMemoryWorkerProcess
+    // throws after its wait loop, so the flag must stay disabled.
+    (Bun as { spawn: typeof Bun.spawn }).spawn = (() => ({
+      unref: () => {},
+      pid: 0,
+    })) as unknown as typeof Bun.spawn;
+    try {
+      const { exitCode, stdout } = await runCommand([
+        "memory",
+        "worker",
+        "start",
+        "--json",
+      ]);
+      expect(exitCode).toBe(1);
+      expect(JSON.parse(stdout)).toMatchObject({ ok: false });
+      expect(workerEnabledFromRaw()).toBe(false);
+    } finally {
+      (Bun as { spawn: typeof Bun.spawn }).spawn = originalSpawn;
+      restore();
+    }
+  });
+});

package/src/cli/commands/memory/index.ts

@@ -4,6 +4,7 @@ import { registerCommand } from "../../lib/register-command.js";
 import { registerMemoryRetrospectiveCommand } from "./memory-retrospective.js";
 import { registerMemoryV2Command } from "./memory-v2.js";
 import { registerMemoryV3Command } from "./memory-v3.js";
+import { registerMemoryWorkerCommand } from "./worker.js";
 
 export function registerMemoryCommand(program: Command): void {
   registerCommand(program, {
@@ -27,6 +28,7 @@ Examples:
       registerMemoryV2Command(memory);
       registerMemoryV3Command(memory);
       registerMemoryRetrospectiveCommand(memory);
+      registerMemoryWorkerCommand(memory);
     },
   });
 }

package/src/cli/commands/memory/worker.ts

@@ -0,0 +1,242 @@
+/**
+ * `assistant memory worker` CLI subgroup.
+ *
+ * Manages the memory jobs worker as its own OS process — separate from the
+ * daemon's main event loop. This prevents long-running embedding jobs from
+ * blocking user-facing HTTP traffic.
+ *
+ * Subcommands:
+ *
+ *   - `start`  — spawn the worker process and enable `memory.worker.enabled`,
+ *     standing the daemon's synchronous in-process runner down.
+ *   - `stop`   — SIGTERM the worker process and disable `memory.worker.enabled`,
+ *     handing the queue back to the synchronous in-process runner.
+ *   - `status` — report the worker process state, the `memory.worker.enabled`
+ *     config value, and whether the synchronous in-process runner is going.
+ *
+ * All three run directly in the CLI process (transport: "local") — no IPC
+ * round-trip to the daemon. The daemon's worker supervisor re-reads
+ * `memory.worker.enabled` each poll and stands its synchronous runner down (or
+ * resumes it) accordingly, so flipping the flag here switches the running
+ * daemon's mode without a restart.
+ */
+
+import type { Command } from "commander";
+
+import {
+  getConfigReadOnly,
+  loadRawConfig,
+  saveRawConfig,
+  setNestedValue,
+} from "../../../config/loader.js";
+import {
+  probeMemoryWorker,
+  probeSyncRunner,
+  spawnMemoryWorkerProcess,
+} from "../../../memory/worker-control.js";
+import { getMemoryWorkerPidPath } from "../../../util/platform.js";
+import { registerCommand } from "../../lib/register-command.js";
+import { log } from "../../logger.js";
+import { shouldOutputJson, writeOutput } from "../../output.js";
+
+/**
+ * Persist `memory.worker.enabled` to the on-disk config via the shared
+ * raw-config helpers, so only this leaf changes (schema defaults are not baked
+ * into the file). The assistant picks the change up on its next config read.
+ */
+function setWorkerEnabled(enabled: boolean): void {
+  const raw = loadRawConfig();
+  setNestedValue(raw, "memory.worker.enabled", enabled);
+  saveRawConfig(raw);
+}
+
+// ---------------------------------------------------------------------------
+// `start`
+// ---------------------------------------------------------------------------
+
+async function startWorker(
+  opts: { json?: boolean },
+  cmd: Command,
+): Promise<void> {
+  let result: { pid: number; alreadyRunning: boolean };
+  try {
+    result = await spawnMemoryWorkerProcess();
+  } catch (err) {
+    // Spawn failed — leave `memory.worker.enabled` untouched so the daemon's
+    // synchronous runner keeps draining the queue rather than standing down
+    // for a worker that never came up.
+    const msg = err instanceof Error ? err.message : String(err);
+    if (shouldOutputJson(cmd)) {
+      writeOutput(cmd, { ok: false, error: msg });
+    } else {
+      log.error(msg);
+    }
+    process.exitCode = 1;
+    return;
+  }
+
+  // The worker process is up (freshly spawned or already running). Enable the
+  // flag so the daemon's supervisor stands its synchronous runner down (and so
+  // the daemon spawns the worker again on the next restart).
+  setWorkerEnabled(true);
+
+  if (shouldOutputJson(cmd)) {
+    writeOutput(cmd, {
+      ok: true,
+      pid: result.pid,
+      alreadyRunning: result.alreadyRunning,
+      pidPath: getMemoryWorkerPidPath(),
+      workerEnabled: true,
+    });
+  } else {
+    log.info(
+      result.alreadyRunning
+        ? `Memory worker is already running (PID ${result.pid})`
+        : `Memory worker started (PID ${result.pid})`,
+    );
+    log.info(
+      "Enabled memory.worker.enabled; the synchronous in-process runner will stand down",
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// `stop`
+// ---------------------------------------------------------------------------
+
+function stopWorker(opts: { json?: boolean }, cmd: Command): void {
+  // Persist the preference first: `stop` means "hand the queue back to the
+  // synchronous in-process runner." Disabling the flag makes the daemon's
+  // supervisor resume processing in-process on its next poll and lines the next
+  // daemon restart up with synchronous mode; the SIGTERM below then stops the
+  // now-redundant worker process.
+  setWorkerEnabled(false);
+
+  const current = probeMemoryWorker();
+  if (current.status !== "running" || current.pid == null) {
+    // No worker process to signal — flipping the flag alone restores
+    // synchronous mode, so this is success, not an error.
+    if (shouldOutputJson(cmd)) {
+      writeOutput(cmd, {
+        ok: true,
+        workerWasRunning: false,
+        workerEnabled: false,
+      });
+    } else {
+      log.info(
+        "Memory worker process was not running; disabled memory.worker.enabled (synchronous runner active)",
+      );
+    }
+    return;
+  }
+
+  const pid = current.pid;
+  try {
+    process.kill(pid, "SIGTERM");
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (shouldOutputJson(cmd)) {
+      writeOutput(cmd, { ok: false, error: msg, pid, workerEnabled: false });
+    } else {
+      log.error(`Failed to stop memory worker (PID ${pid}): ${msg}`);
+    }
+    process.exitCode = 1;
+    return;
+  }
+
+  if (shouldOutputJson(cmd)) {
+    writeOutput(cmd, { ok: true, pid, workerEnabled: false });
+  } else {
+    log.info(`Memory worker stop signal sent (PID ${pid})`);
+    log.info(
+      "Disabled memory.worker.enabled; the synchronous in-process runner will take over",
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// `status`
+// ---------------------------------------------------------------------------
+
+function statusWorker(opts: { json?: boolean }, cmd: Command): void {
+  const worker = probeMemoryWorker();
+  const syncRunner = probeSyncRunner();
+  const workerEnabled = getConfigReadOnly().memory.worker.enabled;
+
+  if (shouldOutputJson(cmd)) {
+    writeOutput(cmd, { ...worker, workerEnabled, syncRunner });
+  } else {
+    if (worker.status === "running") {
+      log.info(`Memory worker process is running (PID ${worker.pid})`);
+    } else {
+      log.info("Memory worker process is not running");
+    }
+    log.info(`memory.worker.enabled: ${workerEnabled}`);
+    if (syncRunner.status === "running") {
+      log.info(
+        `Synchronous in-process runner is running (PID ${syncRunner.pid})`,
+      );
+    } else {
+      log.info("Synchronous in-process runner is not running");
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Registration
+// ---------------------------------------------------------------------------
+
+export function registerMemoryWorkerCommand(memory: Command): void {
+  registerCommand(memory, {
+    name: "worker",
+    transport: "local",
+    description: "Manage the memory jobs worker process (start/stop/status)",
+    build: (worker) => {
+      worker.addHelpText(
+        "after",
+        `
+The memory worker processes embedding, consolidation, and cleanup jobs in a
+separate OS process so they do not block the assistant's main event loop.
+
+\`start\` enables memory.worker.enabled and \`stop\` disables it, so the
+assistant's synchronous in-process runner stands down (start) or takes back over
+(stop) without a restart.
+
+Examples:
+  $ assistant memory worker start
+  $ assistant memory worker status
+  $ assistant memory worker stop`,
+      );
+
+      worker
+        .command("start")
+        .description(
+          "Start the memory worker process and enable memory.worker.enabled",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .action(async (opts: { json?: boolean }, cmd: Command) => {
+          await startWorker(opts, cmd);
+        });
+
+      worker
+        .command("stop")
+        .description(
+          "Stop the memory worker process and disable memory.worker.enabled",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .action((opts: { json?: boolean }, cmd: Command) => {
+          stopWorker(opts, cmd);
+        });
+
+      worker
+        .command("status")
+        .description(
+          "Report worker process state, memory.worker.enabled, and the synchronous runner",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .action((opts: { json?: boolean }, cmd: Command) => {
+          statusWorker(opts, cmd);
+        });
+    },
+  });
+}