@vellumai/assistant 0.11.7-dev.202608281503.cbafd6c → 0.11.7-dev.202608282003.8ce5421

package/src/util/__tests__/worker-process-command.test.ts

@@ -1,6 +1,12 @@
 import { describe, expect, test } from "bun:test";
 
-import { resolveWorkerCommand } from "../worker-process.js";
+import {
+  classifyOrphanAfterWait,
+  decideWorkerSlot,
+  resolveWorkerCommand,
+  workerKindSignature,
+  type WorkerProcessStatus,
+} from "../worker-process.js";
 
 const entry = new URL("file:///source/monitoring/worker.ts");
 
@@ -35,3 +41,242 @@ describe("resolveWorkerCommand", () => {
     ).toEqual(["bun", "--smol", "run", "/source/monitoring/worker.ts"]);
   });
 });
+
+describe("workerKindSignature", () => {
+  const schedule = new URL(
+    "file:////app/runtime/0.11.7/src/schedule/worker.ts",
+  );
+  const source = {
+    platform: "darwin" as const,
+    execPath: "/runtime/bun",
+    executableExists: () => false,
+  };
+
+  test("is the same for a worker of this kind from any install", () => {
+    const previous = new URL(
+      "file:////app/runtime/0.10.11/src/schedule/worker.ts",
+    );
+    expect(workerKindSignature(schedule, "schedule", source)).toEqual(
+      workerKindSignature(previous, "schedule", source),
+    );
+  });
+
+  test("distinguishes one worker kind from another", () => {
+    const monitoring = new URL(
+      "file:////app/runtime/0.11.7/src/monitoring/worker.ts",
+    );
+    expect(workerKindSignature(schedule, "schedule", source)).not.toEqual(
+      workerKindSignature(monitoring, "monitoring", source),
+    );
+  });
+
+  // The signature only guards against signalling a recycled PID, so it has to
+  // be specific enough that an unrelated program running its own worker.ts
+  // never matches.
+  const matches = (command: string, signature: readonly string[]): boolean =>
+    signature.every((part) => command.includes(part));
+
+  test("does not match an unrelated program running some other worker.ts", () => {
+    const signature = workerKindSignature(schedule, "schedule", source);
+    expect(matches("bun run /home/dev/side-project/worker.ts", signature)).toBe(
+      false,
+    );
+    expect(matches("node /srv/queue/src/jobs/worker.ts", signature)).toBe(
+      false,
+    );
+  });
+
+  test("matches the same worker from a previous install", () => {
+    const signature = workerKindSignature(schedule, "schedule", source);
+    expect(
+      matches(
+        "bun --smol run /app/runtime/0.10.11/src/schedule/worker.ts",
+        signature,
+      ),
+    ).toBe(true);
+  });
+
+  const packaged = {
+    platform: "win32" as const,
+    execPath: "/runtime/vellum-daemon.exe",
+    executableExists: () => true,
+  };
+
+  // Every packaged worker runs one executable and is told apart by the
+  // subcommand, so the executable alone must not identify a worker: one
+  // worker's slot would otherwise reclaim another's process.
+  test("distinguishes packaged worker kinds sharing one executable", () => {
+    const scheduleSig = workerKindSignature(schedule, "schedule", packaged);
+    expect(matches('"C:/App/vellum-worker.exe" schedule', scheduleSig)).toBe(
+      true,
+    );
+    expect(matches('"C:/App/vellum-worker.exe" monitoring', scheduleSig)).toBe(
+      false,
+    );
+  });
+
+  test("matches a packaged worker from a previous install", () => {
+    const scheduleSig = workerKindSignature(schedule, "schedule", packaged);
+    expect(matches('"C:/Prev/vellum-worker.exe" schedule', scheduleSig)).toBe(
+      true,
+    );
+  });
+});
+
+// Every branch that can reach a kill. `reclaim` is the only decision that
+// signals a process, so each test below is really asking: could this input
+// have killed something it should not have?
+describe("decideWorkerSlot", () => {
+  const DAEMON = 900;
+  const WORKER = 4242;
+  const signature = ["src/schedule/worker.ts"];
+  const running = { status: "running" as const, pid: WORKER };
+  const ours = `bun --smol run /app/runtime/0.11.7/src/schedule/worker.ts`;
+  const previous = `bun --smol run /app/runtime/0.10.11/src/schedule/worker.ts`;
+  const alive = () => true;
+  const dead = () => false;
+
+  const decide = (
+    row: { pid: number; ppid: number; command: string } | null,
+    isOwnerAlive = alive,
+    pid1OwnsWorkers = false,
+    status: WorkerProcessStatus = running,
+  ) =>
+    decideWorkerSlot(
+      status,
+      row,
+      signature,
+      DAEMON,
+      isOwnerAlive,
+      pid1OwnsWorkers,
+    );
+
+  test("spawns when the PID file names nothing running", () => {
+    expect(decide(null, alive, false, { status: "not_running" })).toEqual({
+      action: "spawn",
+    });
+  });
+
+  test("adopts this daemon's own worker", () => {
+    expect(decide({ pid: WORKER, ppid: DAEMON, command: ours })).toEqual({
+      action: "adopt",
+      pid: WORKER,
+    });
+  });
+
+  test("reclaims a worker reparented to init after its daemon died", () => {
+    expect(decide({ pid: WORKER, ppid: 1, command: previous })).toEqual({
+      action: "reclaim",
+      pid: WORKER,
+    });
+  });
+
+  test("reclaims a worker whose owner is gone", () => {
+    expect(decide({ pid: WORKER, ppid: 777, command: previous }, dead)).toEqual(
+      {
+        action: "reclaim",
+        pid: WORKER,
+      },
+    );
+  });
+
+  // `isOwnerAlive` is "is the owner a daemon", not "does that PID exist". These
+  // workers have exactly one legitimate owner, so a recycled owner PID must not
+  // keep a stranded worker looking owned.
+  test("adopts a worker a live daemon owns", () => {
+    expect(decide({ pid: WORKER, ppid: 777, command: ours }, alive)).toEqual({
+      action: "adopt",
+      pid: WORKER,
+    });
+  });
+
+  test("reclaims a worker whose owner PID was recycled to a non-daemon", () => {
+    const ownerIsNotADaemon = () => false;
+    expect(
+      decide({ pid: WORKER, ppid: 777, command: previous }, ownerIsNotADaemon),
+    ).toEqual({ action: "reclaim", pid: WORKER });
+  });
+
+  // In a container the daemon is PID 1, so a worker parented to 1 is a live
+  // sibling's, not an orphan.
+  test("adopts a PID-1 child when PID 1 is the daemon", () => {
+    expect(
+      decide({ pid: WORKER, ppid: 1, command: ours }, alive, true),
+    ).toEqual({ action: "adopt", pid: WORKER });
+  });
+
+  // The cases below are the ones that must never reach a kill.
+  test("never signals an unrelated process on a recycled PID", () => {
+    expect(
+      decide({ pid: WORKER, ppid: 1, command: "/usr/bin/postgres -D /data" }),
+    ).toEqual({ action: "adopt", pid: WORKER });
+  });
+
+  test("never signals another project's worker.ts on a recycled PID", () => {
+    expect(
+      decide({
+        pid: WORKER,
+        ppid: 1,
+        command: "bun run /home/dev/side-project/worker.ts",
+      }),
+    ).toEqual({ action: "adopt", pid: WORKER });
+  });
+
+  test("never signals a different worker kind holding this slot", () => {
+    expect(
+      decide({
+        pid: WORKER,
+        ppid: 1,
+        command: "bun --smol run /app/runtime/0.10.11/src/monitoring/worker.ts",
+      }),
+    ).toEqual({ action: "adopt", pid: WORKER });
+  });
+
+  test("never signals when the process table could not be read", () => {
+    expect(decide(null)).toEqual({ action: "adopt", pid: WORKER });
+  });
+});
+
+// Escalation happens after an awaited SIGTERM grace, so the PID alone cannot
+// carry identity across it: the orphan may exit mid-wait and the OS may hand
+// its PID to a stranger, or the process table may simply fail to answer.
+// Only a positive re-match escalates, and only a proven "gone" releases the
+// slot; a live PID whose identity cannot be read resolves to reuse, because
+// releasing it would spawn a second worker next to a live one.
+describe("classifyOrphanAfterWait", () => {
+  const signature = ["src/schedule/worker.ts"];
+  const ours = "bun --smol run /app/runtime/0.10.11/src/schedule/worker.ts";
+
+  test("dead PID: gone, slot may be released", () => {
+    expect(classifyOrphanAfterWait(false, null, signature)).toBe("gone");
+    expect(classifyOrphanAfterWait(false, ours, signature)).toBe("gone");
+  });
+
+  test("still this worker: escalation proceeds", () => {
+    expect(classifyOrphanAfterWait(true, ours, signature)).toBe("orphan");
+  });
+
+  test("PID reused by a stranger: orphan exited, never SIGKILL", () => {
+    expect(
+      classifyOrphanAfterWait(true, "/usr/bin/postgres -D /data", signature),
+    ).toBe("gone");
+  });
+
+  test("PID reused by a different worker kind: never SIGKILL", () => {
+    expect(
+      classifyOrphanAfterWait(
+        true,
+        "bun --smol run /app/runtime/0.10.11/src/monitoring/worker.ts",
+        signature,
+      ),
+    ).toBe("gone");
+  });
+
+  // The fail-safe vex pinned: a live PID with an unreadable command line is
+  // neither killable nor releasable.
+  test("alive but command unreadable: identity-unreadable, reuse the slot", () => {
+    expect(classifyOrphanAfterWait(true, null, signature)).toBe(
+      "identity-unreadable",
+    );
+  });
+});

package/src/util/platform.ts

@@ -1,14 +1,6 @@
-import { chmodSync, existsSync, mkdirSync, realpathSync } from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import {
-  basename,
-  delimiter,
-  dirname,
-  isAbsolute,
-  join,
-  relative,
-  sep,
-} from "node:path";
+import { chmodSync, existsSync, mkdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { delimiter, dirname, isAbsolute, join, relative, sep } from "node:path";
 
 import {
   AVATAR_IMAGE_FILENAME,
@@ -16,6 +8,7 @@ import {
   resolveAvatarDir,
 } from "@vellumai/avatar-manifest";
 import { SEEDS } from "@vellumai/environments";
+import { assertTestPathIsEphemeral as assertEphemeralInTests } from "@vellumai/environments/test-path-guard";
 
 import { getWorkspaceDirOverride } from "../config/env-registry.js";
 
@@ -405,78 +398,17 @@ export function getProcPidPath(name: string): string {
 //
 // A test process must never resolve the workspace (or the vellum root) to a
 // real, non-temp directory: production code exercised by a test would then
-// read and destructively write live state. The tmpdir redirection normally
-// comes from the bunfig.toml test preload, but bun only loads bunfig from the
-// cwd, so `bun test` run from any other directory (for example a source
-// checkout inside a deployed container's workspace) skips the preload and
-// inherits the ambient VELLUM_WORKSPACE_DIR. The containment assertion
-// therefore lives here, in production code, where it fires no matter how the
-// test process was launched.
-//
-// Containment logic mirrors src/__tests__/assert-not-live-db.ts, which cannot
-// be imported here (production code must not depend on test machinery).
-
-/** Lazily computed: is this process a `bun test` run? */
-let isTestProcess: boolean | undefined;
-
-/**
- * Resolve symlinks in the deepest existing ancestor of `p`, then re-append
- * the not-yet-created tail. This keeps the containment check honest both for
- * paths under a symlinked temp root (macOS /var/folders) and for symlinks
- * that point outside it, whether or not the leaf exists yet.
- */
-function canonicalizeForWorkspaceGuard(p: string): string {
-  let cur = p;
-  const tail: string[] = [];
-  for (;;) {
-    try {
-      return join(realpathSync(cur), ...tail);
-    } catch {
-      const parent = dirname(cur);
-      if (parent === cur) {
-        return p;
-      }
-      tail.unshift(basename(cur));
-      cur = parent;
-    }
-  }
-}
+// read and destructively write live state. The containment assertion is
+// shared with the gateway via @vellumai/environments/test-path-guard;
+// src/__tests__/assert-not-live-db.ts keeps its own containment check
+// because test machinery must not import production dependencies.
 
 function assertTestPathIsEphemeral(dir: string): void {
-  isTestProcess ??=
-    process.env.NODE_ENV === "test" ||
-    process.env.BUN_TEST === "1" ||
-    // `bun test` sets NODE_ENV=test only when unset; Bun.main being the test
-    // file itself is the backstop signal that survives a preset NODE_ENV.
-    (typeof Bun !== "undefined" &&
-      /\.(test|spec)\.[cm]?[jt]sx?$/.test(Bun.main));
-  if (!isTestProcess) {
-    return;
-  }
-  // Escape hatch for the rare intentional run against a real workspace,
-  // shared with assertTestDbIsIsolated() in persistence/db-connection.ts.
-  // Deliberately NOT added to tools/terminal/safe-env.ts: a daemon-level
-  // opt-out must not propagate into agent-spawned shells and disarm the
-  // guard for tests run from there.
-  if (process.env.VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS === "1") {
-    return;
-  }
-  const tmpRoot = canonicalizeForWorkspaceGuard(tmpdir());
-  const resolved = canonicalizeForWorkspaceGuard(dir);
-  if (resolved !== tmpRoot && !resolved.startsWith(tmpRoot + sep)) {
-    throw new Error(
-      [
-        `Refusing to use ${dir} (resolves to ${resolved}) in a test process: it is not under the temp directory (${tmpRoot}).`,
-        "",
-        "Tests must only touch an ephemeral workspace; a real one would expose",
-        "live assistant state to destructive test fixtures. This usually means",
-        "`bun test` ran from a cwd without the repo bunfig.toml, so the test",
-        "preload that redirects VELLUM_WORKSPACE_DIR to a tmpdir never loaded.",
-        "Run tests from the assistant package root, or set",
-        "VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS=1 to bypass deliberately.",
-      ].join("\n"),
-    );
-  }
+  assertEphemeralInTests(dir, {
+    // Shared with assertTestDbIsIsolated() in persistence/db-connection.ts.
+    allowEnvVar: "VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS",
+    runHint: "Run tests from the assistant package root.",
+  });
 }
 
 /**

package/src/util/process-table.ts

@@ -231,6 +231,19 @@ export async function listProcessTableAsync(
   );
 }
 
+/**
+ * The process-table row for `pid`, or null when the process is gone or the
+ * table cannot be read. Callers get `ppid` alongside the command line, so
+ * ownership and identity can be decided from a single snapshot.
+ */
+function findProcessRow(pid: number): ProcessTableRow | null {
+  try {
+    return listProcessTable().find((row) => row.pid === pid) ?? null;
+  } catch {
+    return null;
+  }
+}
+
 export function readRawProcessCommand(pid: number): string | null {
   if (process.platform === "linux") {
     try {
@@ -243,9 +256,5 @@ export function readRawProcessCommand(pid: number): string | null {
     }
   }
 
-  try {
-    return listProcessTable().find((row) => row.pid === pid)?.command ?? null;
-  } catch {
-    return null;
-  }
+  return findProcessRow(pid)?.command ?? null;
 }

package/src/util/worker-ownership.ts

@@ -0,0 +1,126 @@
+/**
+ * Process-tree ownership for background worker processes.
+ *
+ * A worker's parent IS its owner. The process table is therefore the
+ * authoritative answer to "whose worker is this": unlike a PID file it
+ * survives a crash, and a second owner cannot overwrite it.
+ *
+ * Nothing here is specific to one worker kind. `listWorkerProcesses` matches
+ * on the absolute worker path the caller passes, and `classifyWorkerOwnership`
+ * takes injected liveness and PID-1 predicates, so any worker whose spawner
+ * knows its own PID can use both.
+ */
+
+import { listProcessTable, readRawProcessCommand } from "./process-table.js";
+
+export interface WorkerProcess {
+  pid: number;
+  ppid: number;
+}
+
+/**
+ * What this process may do with a worker it found in the process table.
+ *
+ * - `reclaim`: parented to us, a worker we started and lost track of. Reaping
+ *   it is what enforces one live worker per owning process.
+ * - `orphan`: reparented to init, or its owner is gone. Nobody else will ever
+ *   clean it up.
+ * - `foreign`: owned by another live process. The memory-worker process runs
+ *   its own backend against this same workspace and is entitled to its own
+ *   worker; signalling it is what made the two replace each other in a loop.
+ *
+ * A parent of PID 1 is ambiguous, and resolving it wrongly in either direction
+ * costs something real. `docker-entrypoint.sh` execs the daemon, so PID 1 can
+ * BE the daemon and its child a healthy sibling's worker. Under `docker run
+ * --init`, and on any host, PID 1 is an init process instead, so the same
+ * parentage means the owner died and the worker needs reclaiming. Callers
+ * therefore pass what PID 1 actually is rather than inferring it from whether
+ * the deployment is containerized, which is true in both container shapes.
+ */
+export type WorkerOwnership = "reclaim" | "orphan" | "foreign";
+
+export function classifyWorkerOwnership(
+  worker: WorkerProcess,
+  selfPid: number,
+  isOwnerAlive: (pid: number) => boolean,
+  pid1OwnsWorkers: boolean,
+): WorkerOwnership {
+  if (worker.ppid === selfPid) {
+    return "reclaim";
+  }
+  if (worker.ppid <= 1) {
+    return pid1OwnsWorkers ? "foreign" : "orphan";
+  }
+  if (!isOwnerAlive(worker.ppid)) {
+    return "orphan";
+  }
+  return "foreign";
+}
+
+/**
+ * Command lines that identify an assistant daemon, running from source
+ * (`.../daemon/main.ts`) or as the packaged binary.
+ */
+const DAEMON_PROCESS_PATTERN = /vellum-daemon|[\\/]daemon[\\/]main/;
+
+/**
+ * Whether a command line belongs to an assistant daemon.
+ *
+ * For a worker with exactly one legitimate owner, "is the owner alive" is not
+ * the same question as "does some process still hold that PID". The OS recycles
+ * PIDs, and a recycled owner PID would otherwise leave a stranded worker
+ * looking owned, and therefore untouchable, indefinitely.
+ */
+export function isDaemonCommand(command: string | null): boolean {
+  return command != null && DAEMON_PROCESS_PATTERN.test(command);
+}
+
+/**
+ * Whether PID 1 is an assistant daemon rather than an init process.
+ *
+ * True only where the daemon was exec'd as PID 1 (`docker-entrypoint.sh`),
+ * which is what makes a worker parented to 1 a live sibling's rather than an
+ * orphan. Under `docker run --init` PID 1 is docker-init and under launchd or
+ * systemd it is the system init, so both answer false and PID-1 orphans stay
+ * reclaimable. Unreadable means false, which keeps the reclaiming behaviour.
+ *
+ * Callers that already hold a process-table snapshot pass PID 1's command line
+ * so this does not read the table a second time.
+ */
+export function pid1OwnsWorkers(pid1Command?: string | null): boolean {
+  if (process.pid === 1) {
+    return true;
+  }
+  return isDaemonCommand(
+    pid1Command === undefined ? readRawProcessCommand(1) : pid1Command,
+  );
+}
+
+/**
+ * Live worker processes matching `workerPath`, as `(pid, ppid)` pairs.
+ *
+ * `workerPath` is an absolute path unique to one worker of one install, so a
+ * sibling instance's workers never match. `model`, when given, must appear as
+ * a whole argv token rather than a substring, so a backend for `bge-small`
+ * cannot reclaim a live `bge-small-v2` worker.
+ *
+ * Raw command lines are read for matching only, never stored or logged:
+ * process arguments can carry secrets (see the redaction note in
+ * `util/process-tree.ts`).
+ */
+export function listWorkerProcesses(
+  workerPath: string,
+  model?: string,
+): WorkerProcess[] {
+  try {
+    return listProcessTable()
+      .filter(
+        (row) =>
+          row.command.includes(workerPath) &&
+          (!model || row.command.split(/\s+/).includes(model)),
+      )
+      .map(({ pid, ppid }) => ({ pid, ppid }));
+  } catch {
+    return [];
+  }
+}