@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/util/worker-process.ts

@@ -0,0 +1,271 @@
+/**
+ * Generic control surface for a background worker OS process tracked via a
+ * PID file: probe liveness, spawn an entry script and wait for readiness,
+ * and signal the process to stop.
+ *
+ * Domain modules (memory jobs worker, resource monitor, schedule worker) wrap
+ * these helpers with their own PID path, entry point, and error type — the
+ * PID-file bookkeeping lives here in one place.
+ */
+
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  unlinkSync,
+} from "node:fs";
+import { dirname } from "node:path";
+
+import { getCurrentLogFilePath } from "./logger.js";
+
+export interface WorkerProcessStatus {
+  status: "running" | "not_running";
+  pid?: number;
+}
+
+/** True when `err` is a Node ESRCH error ("no such process"). */
+function isEsrchError(err: unknown): boolean {
+  return (
+    !!err &&
+    typeof err === "object" &&
+    "code" in err &&
+    (err as { code?: unknown }).code === "ESRCH"
+  );
+}
+
+/**
+ * Read a PID file and report liveness. A missing or malformed file reports
+ * not_running; a file pointing at a dead process is cleaned up and reported as
+ * not_running. Intended for worker-process PID files whose PID is a normal
+ * spawned child (never PID 1), so `process.kill(pid, 0)` liveness is reliable.
+ */
+export function probeWorkerPidFile(path: string): WorkerProcessStatus {
+  if (!existsSync(path)) {
+    return { status: "not_running" };
+  }
+
+  const raw = readFileSync(path, "utf-8").trim();
+  const pid = parseInt(raw, 10);
+  if (!Number.isFinite(pid) || pid <= 0) {
+    return { status: "not_running" };
+  }
+
+  try {
+    process.kill(pid, 0);
+    return { status: "running", pid };
+  } catch (err: unknown) {
+    if (isEsrchError(err)) {
+      // Stale file — clean it up.
+      try {
+        unlinkSync(path);
+      } catch {
+        // best-effort
+      }
+      return { status: "not_running" };
+    }
+    // Any other error (e.g. EPERM: the process exists but this caller may not
+    // signal it) means the process is alive. Report it running rather than
+    // letting the error escape a status probe.
+    return { status: "running", pid };
+  }
+}
+
+/** Thrown when a worker process fails to come up within the wait window. */
+export class WorkerProcessSpawnError extends Error {}
+
+/**
+ * How long {@link spawnWorkerProcess} waits for the freshly-spawned worker to
+ * write its PID file before treating the spawn as failed. A cold `bun run`
+ * start — new runtime, config load, DB open — routinely takes several
+ * seconds, so this is deliberately generous: a premature timeout makes a
+ * `start` command report failure for a worker that is merely slow.
+ */
+const PID_FILE_WAIT_TIMEOUT_MS = 15_000;
+
+/** Poll interval while waiting for the worker's PID file to appear. */
+const PID_FILE_POLL_INTERVAL_MS = 100;
+
+export interface SpawnWorkerProcessOptions {
+  /**
+   * Override how long to wait for the worker's PID file, in ms. Defaults to
+   * {@link PID_FILE_WAIT_TIMEOUT_MS}. Primarily a testing seam.
+   */
+  pidWaitTimeoutMs?: number;
+  /**
+   * Override the PID-file poll interval, in ms. Defaults to
+   * {@link PID_FILE_POLL_INTERVAL_MS}. Primarily a testing seam.
+   */
+  pidPollIntervalMs?: number;
+  /**
+   * When the wait times out while the child is still alive (a hung or very
+   * slow start), terminate that child before throwing. Callers whose failure
+   * path leaves the worker's config flag off MUST set this: otherwise the
+   * detached worker keeps coming up and runs alongside whatever in-process
+   * fallback the flag re-enabled.
+   */
+  terminateOnTimeout?: boolean;
+  /**
+   * Process parentage (default `true`):
+   *   - `true` — the worker is its own session leader and outlives the
+   *     spawning process. Short-lived CLI spawners need this so the worker
+   *     keeps running after the command returns.
+   *   - `false` — the worker is a direct child of the spawning process. The
+   *     daemon passes this so the worker it owns appears in its process tree
+   *     (`assistant ps`) and is torn down with the daemon.
+   * Either way the child is `unref`'d, so the spawning process never blocks on
+   * it and the worker is tracked via its PID file rather than the handle.
+   */
+  detached?: boolean;
+}
+
+type WorkerReadyOutcome = "ready" | "exited" | "timeout";
+
+/**
+ * Wait for the worker to signal readiness by writing its PID file.
+ *
+ *   - `"ready"`   — the PID file appeared.
+ *   - `"exited"`  — the child exited first (it crashed during startup).
+ *   - `"timeout"` — neither happened within `timeoutMs`.
+ *
+ * Polls for the PID file but also watches `exited`, so a crash-on-startup
+ * fails fast instead of waiting out the whole timeout.
+ */
+async function waitForWorkerPidFile(
+  pidPath: string,
+  exited: Promise<unknown> | undefined,
+  timeoutMs: number,
+  pollIntervalMs: number,
+): Promise<WorkerReadyOutcome> {
+  let childExited = false;
+  const markExited = () => {
+    childExited = true;
+  };
+  // A pending `exited` promise does not keep the event loop alive once the
+  // child is unref'd, so this floating wait won't hang the process.
+  void exited?.then(markExited, markExited);
+
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (existsSync(pidPath)) {
+      return "ready";
+    }
+    if (childExited) {
+      // The worker could write the PID file and exit in the same tick —
+      // re-check before declaring it dead.
+      return existsSync(pidPath) ? "ready" : "exited";
+    }
+    await Bun.sleep(pollIntervalMs);
+  }
+  return existsSync(pidPath) ? "ready" : "timeout";
+}
+
+/**
+ * Spawn a worker entry script as a background process and wait for it to
+ * report readiness by writing its PID file. `opts.detached` (default `true`)
+ * controls process parentage — see {@link SpawnWorkerProcessOptions}.
+ *
+ * If a worker is already running (per the PID file), returns its PID with
+ * `alreadyRunning: true` rather than spawning a second one. Throws
+ * {@link WorkerProcessSpawnError} if the child crashes during startup or
+ * never writes its PID file within the wait window.
+ */
+export async function spawnWorkerProcess(args: {
+  /** PID file the worker writes on startup (and that probes read). */
+  pidPath: string;
+  /** Worker entry script, e.g. `new URL("./worker.ts", import.meta.url)`. */
+  entry: URL;
+  /** Human-readable name used in spawn-failure messages, e.g. "Memory worker". */
+  workerLabel: string;
+  options?: SpawnWorkerProcessOptions;
+}): Promise<{ pid: number; alreadyRunning: boolean }> {
+  const opts = args.options ?? {};
+  const pidWaitTimeoutMs = opts.pidWaitTimeoutMs ?? PID_FILE_WAIT_TIMEOUT_MS;
+  const pidPollIntervalMs = opts.pidPollIntervalMs ?? PID_FILE_POLL_INTERVAL_MS;
+  const detached = opts.detached ?? true;
+
+  const current = probeWorkerPidFile(args.pidPath);
+  if (current.status === "running" && current.pid != null) {
+    return { pid: current.pid, alreadyRunning: true };
+  }
+
+  // Pipe the worker's stderr into the same daily log file the daemon writes
+  // to. The worker's pino logger already writes there directly, but stderr
+  // captures crash traces (uncaught exceptions that bypass the catch handler)
+  // and pino's fallback output if the file logger fails to initialize.
+  // Without this, any such output is lost to /dev/null and the worker dies
+  // silently.
+  let stderrFd: number | "inherit" = "inherit";
+  try {
+    const logPath = getCurrentLogFilePath();
+    mkdirSync(dirname(logPath), { recursive: true });
+    stderrFd = openSync(logPath, "a", 0o600);
+  } catch {
+    // If the log file can't be opened, inherit the parent's stderr so crash
+    // output is at least visible to the spawning process.
+  }
+
+  const child = Bun.spawn({
+    cmd: ["bun", "run", args.entry.pathname],
+    stdio: ["ignore", "ignore", stderrFd],
+    detached,
+  });
+
+  // Close our copy of the log fd — the child has its own.
+  if (typeof stderrFd === "number") {
+    closeSync(stderrFd);
+  }
+
+  // Unreference so the spawning process doesn't wait for the child.
+  child.unref();
+
+  // The child is `unref`'d, so a worker that is merely slow keeps coming up
+  // after we stop waiting.
+  const outcome = await waitForWorkerPidFile(
+    args.pidPath,
+    child.exited,
+    pidWaitTimeoutMs,
+    pidPollIntervalMs,
+  );
+
+  if (outcome !== "ready") {
+    // On a plain timeout the child may still be alive (hung or very slow
+    // start). Terminate it when asked so a worker we are reporting as failed
+    // cannot come up later. On an early exit the child is already gone, so
+    // there is nothing to kill.
+    if (outcome === "timeout" && opts.terminateOnTimeout) {
+      try {
+        child.kill();
+      } catch {
+        // best-effort — the child may already be gone
+      }
+    }
+    throw new WorkerProcessSpawnError(
+      outcome === "exited"
+        ? `${args.workerLabel} exited during startup before writing its PID file`
+        : `${args.workerLabel} was spawned but did not write its PID file within ${Math.round(
+            pidWaitTimeoutMs / 1000,
+          )}s`,
+    );
+  }
+
+  const pid = parseInt(readFileSync(args.pidPath, "utf-8").trim(), 10);
+  return { pid, alreadyRunning: false };
+}
+
+/**
+ * Send SIGTERM to the worker process behind `pidPath` if it is actually
+ * running.
+ *
+ * Returns the status observed before signalling, so callers can report
+ * whether anything was stopped. Only throws if `process.kill` itself fails
+ * (e.g. EPERM) — a not-running worker is a no-op.
+ */
+export function stopWorkerProcess(pidPath: string): WorkerProcessStatus {
+  const current = probeWorkerPidFile(pidPath);
+  if (current.status === "running" && current.pid != null) {
+    process.kill(current.pid, "SIGTERM");
+  }
+  return current;
+}

package/src/workflows/journal-store.ts

@@ -217,6 +217,7 @@ export function createRun(input: CreateRunInput): WorkflowRun {
   const now = Date.now();
   const status = input.status ?? "running";
   rawRun(
+    "journal:createRun",
     /*sql*/ `
     INSERT INTO workflow_runs (
       id, name, script_source, script_hash, args_json, capabilities_json,
@@ -275,7 +276,11 @@ export function updateRun(
   params.push(Date.now());
   params.push(runId);
 
-  rawRun(`UPDATE workflow_runs SET ${sets.join(", ")} WHERE id = ?`, ...params);
+  rawRun(
+    "journal:updateRun",
+    `UPDATE workflow_runs SET ${sets.join(", ")} WHERE id = ?`,
+    ...params,
+  );
   return getRun(runId);
 }
 
@@ -290,6 +295,7 @@ export function finishRun(
 ): WorkflowRun | null {
   const now = Date.now();
   rawRun(
+    "journal:finishRun",
     /*sql*/ `
     UPDATE workflow_runs
     SET status = ?, result_json = ?, error = ?, updated_at = ?, finished_at = ?
@@ -326,6 +332,7 @@ export function appendJournalEntry(
   input: AppendJournalEntryInput,
 ): WorkflowJournalEntry {
   rawRun(
+    "journal:appendJournalEntry",
     /*sql*/ `
     INSERT INTO workflow_journal (
       run_id, seq, call_hash, kind, request_json, result_json, status,
@@ -361,6 +368,7 @@ export function appendJournalEntry(
 /** Fetch a single run by id, or null if it doesn't exist. */
 export function getRun(runId: string): WorkflowRun | null {
   const row = rawGet<WorkflowRunRow>(
+    "journal:getRun",
     `SELECT * FROM workflow_runs WHERE id = ?`,
     runId,
   );
@@ -370,6 +378,7 @@ export function getRun(runId: string): WorkflowRun | null {
 /** Fetch a run's full journal in sequence order for resume replay. */
 export function getJournal(runId: string): WorkflowJournalEntry[] {
   const rows = rawAll<WorkflowJournalRow>(
+    "journal:getJournal",
     `SELECT * FROM workflow_journal WHERE run_id = ? ORDER BY seq ASC`,
     runId,
   );
@@ -382,6 +391,7 @@ export function getJournalEntry(
   seq: number,
 ): WorkflowJournalEntry | null {
   const row = rawGet<WorkflowJournalRow>(
+    "journal:getJournalEntry",
     `SELECT * FROM workflow_journal WHERE run_id = ? AND seq = ?`,
     runId,
     seq,
@@ -398,11 +408,13 @@ export interface ListRunsOptions {
 export function listRuns(options: ListRunsOptions): WorkflowRun[] {
   const rows = options.status
     ? rawAll<WorkflowRunRow>(
+        "journal:listRuns:byStatus",
         `SELECT * FROM workflow_runs WHERE status = ? ORDER BY created_at DESC LIMIT ?`,
         options.status,
         options.limit,
       )
     : rawAll<WorkflowRunRow>(
+        "journal:listRuns:all",
         `SELECT * FROM workflow_runs ORDER BY created_at DESC LIMIT ?`,
         options.limit,
       );
@@ -427,6 +439,7 @@ export function listRuns(options: ListRunsOptions): WorkflowRun[] {
  */
 export function markRunningAsInterrupted(): number {
   return rawRun(
+    "journal:markRunningAsInterrupted",
     /*sql*/ `UPDATE workflow_runs SET status = 'interrupted', updated_at = ? WHERE status = 'running'`,
     Date.now(),
   );
@@ -464,6 +477,7 @@ const TERMINAL_RUN_STATUS_SQL = TERMINAL_RUN_STATUSES.map((s) => `'${s}'`).join(
 export function pruneRuns(retentionDays: number): number {
   const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
   rawRun(
+    "journal:pruneRuns:journal",
     /*sql*/ `
     DELETE FROM workflow_journal
     WHERE run_id IN (
@@ -475,6 +489,7 @@ export function pruneRuns(retentionDays: number): number {
     cutoff,
   );
   return rawRun(
+    "journal:pruneRuns:runs",
     /*sql*/ `
     DELETE FROM workflow_runs
     WHERE status IN (${TERMINAL_RUN_STATUS_SQL})

package/src/workflows/library.test.ts

@@ -218,8 +218,8 @@ describe("getWorkflow", () => {
   });
 });
 
-describe("name collisions", () => {
-  test("a directory wins over a same-base-name flat file", () => {
+describe("name collisions (fail closed)", () => {
+  test("a base-name collision (dir + flat) resolves NEITHER", () => {
     writeWorkflow(
       "foo.workflow.ts",
       `export const meta = { name: "foo", description: "flat" };\nreturn "FLAT";`,
@@ -229,16 +229,12 @@ describe("name collisions", () => {
       `export const meta = { name: "foo", description: "dir" };\nreturn "DIR";`,
     );
 
-    const resolved = getWorkflow("foo")!;
-    expect(resolved.source).toContain("DIR");
-    expect(resolved.path.endsWith(join("foo", "workflow.ts"))).toBe(true);
-
-    const foos = listWorkflows().filter((e) => e.name === "foo");
-    expect(foos).toHaveLength(1);
-    expect(foos[0]!.description).toBe("dir");
+    // Neither the base name nor the (shared) meta.name resolves.
+    expect(getWorkflow("foo")).toBeNull();
+    expect(listWorkflows().filter((e) => e.name === "foo")).toHaveLength(0);
   });
 
-  test("the shadowed flat file's own meta.name no longer resolves", () => {
+  test("a base-name collision hides both sides' own meta.names", () => {
     writeWorkflow(
       "bar.workflow.ts",
       `export const meta = { name: "flat-name", description: "flat" };\nreturn "FLAT";`,
@@ -248,14 +244,33 @@ describe("name collisions", () => {
       `export const meta = { name: "dir-name", description: "dir" };\nreturn "DIR";`,
     );
 
-    expect(getWorkflow("dir-name")!.source).toContain("DIR");
+    expect(getWorkflow("dir-name")).toBeNull();
     expect(getWorkflow("flat-name")).toBeNull();
-    // Base-name fallback also lands on the directory.
-    expect(getWorkflow("bar")!.source).toContain("DIR");
-    expect(listWorkflows().map((e) => e.name)).toEqual(["dir-name"]);
+    expect(getWorkflow("bar")).toBeNull();
+    expect(listWorkflows()).toEqual([]);
+  });
+
+  test("a base-name collision does not affect unrelated workflows", () => {
+    writeWorkflow(
+      "foo.workflow.ts",
+      `export const meta = { name: "foo", description: "flat" };\nreturn "FLAT";`,
+    );
+    writeDirWorkflow(
+      "foo",
+      `export const meta = { name: "foo", description: "dir" };\nreturn "DIR";`,
+    );
+    // A separate, non-colliding workflow still resolves normally.
+    writeWorkflow(
+      "safe.workflow.ts",
+      `export const meta = { name: "safe", description: "ok" };\nreturn "SAFE";`,
+    );
+
+    expect(getWorkflow("safe")!.source).toContain("SAFE");
+    expect(listWorkflows().map((e) => e.name)).toEqual(["safe"]);
   });
 
-  test("listWorkflows dedups a duplicate meta.name across different files", () => {
+  test("a duplicate meta.name across different files resolves NEITHER", () => {
+    // Distinct base names, same meta.name → fail closed (no hijack by order).
     writeWorkflow(
       "a.workflow.ts",
       `export const meta = { name: "dup", description: "from-a" };\nreturn "A";`,
@@ -265,10 +280,10 @@ describe("name collisions", () => {
       `export const meta = { name: "dup", description: "from-b" };\nreturn "B";`,
     );
 
-    // One entry; the directory (yielded first) wins, consistent with getWorkflow.
-    const dups = listWorkflows().filter((e) => e.name === "dup");
-    expect(dups).toHaveLength(1);
-    expect(dups[0]!.description).toBe("from-b");
-    expect(getWorkflow("dup")!.source).toContain("B");
+    expect(getWorkflow("dup")).toBeNull();
+    expect(listWorkflows().filter((e) => e.name === "dup")).toHaveLength(0);
+    // Each file is still reachable by its unambiguous base name.
+    expect(getWorkflow("a")!.source).toContain("A");
+    expect(getWorkflow("b")!.source).toContain("B");
   });
 });