@vellumai/cli 0.8.4 → 0.8.6

package/src/lib/platform-client.ts

@@ -521,6 +521,8 @@ export async function hatchAssistant(
   );
 }
 
+const PLATFORM_FETCH_TIMEOUT_MS = 10_000;
+
 /**
  * Lightweight pre-check: returns the first active managed assistant for the
  * authenticated user, or `null` if none exists. Calls `GET /v1/assistants/`
@@ -536,20 +538,31 @@ export async function checkExistingPlatformAssistant(
   const resolvedUrl = platformUrl || getPlatformUrl();
   const url = `${resolvedUrl}/v1/assistants/`;
 
-  const response = await fetch(url, {
-    headers: await authHeaders(token, platformUrl),
-  });
+  const controller = new AbortController();
+  const timeoutId = setTimeout(
+    () => controller.abort(),
+    PLATFORM_FETCH_TIMEOUT_MS,
+  );
 
-  if (!response.ok) {
-    // Non-fatal: if the list call fails, fall through and let hatch handle it.
-    return null;
-  }
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: await authHeaders(token, platformUrl),
+    });
 
-  const body = (await response.json()) as {
-    results?: HatchedAssistant[];
-  };
-  const active = body.results?.find((a) => a.status === "active");
-  return active ?? null;
+    if (!response.ok) {
+      // Non-fatal: if the list call fails, fall through and let hatch handle it.
+      return null;
+    }
+
+    const body = (await response.json()) as {
+      results?: HatchedAssistant[];
+    };
+    const active = body.results?.find((a) => a.status === "active");
+    return active ?? null;
+  } finally {
+    clearTimeout(timeoutId);
+  }
 }
 
 /**
@@ -563,17 +576,28 @@ export async function fetchPlatformAssistants(
   const resolvedUrl = platformUrl || getPlatformUrl();
   const url = `${resolvedUrl}/v1/assistants/`;
 
-  const response = await fetch(url, {
-    headers: await authHeaders(token, platformUrl),
-  });
+  const controller = new AbortController();
+  const timeoutId = setTimeout(
+    () => controller.abort(),
+    PLATFORM_FETCH_TIMEOUT_MS,
+  );
 
-  if (!response.ok) return [];
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: await authHeaders(token, platformUrl),
+    });
 
-  const body = (await response.json()) as {
-    results?: HatchedAssistant[];
-  };
+    if (!response.ok) return [];
 
-  return (body.results ?? []).filter((a) => a.status === "active");
+    const body = (await response.json()) as {
+      results?: HatchedAssistant[];
+    };
+
+    return (body.results ?? []).filter((a) => a.status === "active");
+  } finally {
+    clearTimeout(timeoutId);
+  }
 }
 
 export interface PlatformUser {
@@ -592,22 +616,34 @@ export async function fetchOrganizationId(
 ): Promise<string> {
   const resolvedUrl = platformUrl || getPlatformUrl();
   const url = `${resolvedUrl}/v1/organizations/`;
-  const response = await fetch(url, {
-    headers: { ...tokenAuthHeader(token) },
-  });
 
-  if (!response.ok) {
-    throw new Error(
-      `Failed to fetch organizations from ${resolvedUrl} (${response.status}). Try logging in again.`,
-    );
-  }
+  const controller = new AbortController();
+  const timeoutId = setTimeout(
+    () => controller.abort(),
+    PLATFORM_FETCH_TIMEOUT_MS,
+  );
 
-  const body = (await response.json()) as OrganizationListResponse;
-  const orgId = body.results?.[0]?.id;
-  if (!orgId) {
-    throw new Error("No organization found for this account.");
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: { ...tokenAuthHeader(token) },
+    });
+
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch organizations from ${resolvedUrl} (${response.status}). Try logging in again.`,
+      );
+    }
+
+    const body = (await response.json()) as OrganizationListResponse;
+    const orgId = body.results?.[0]?.id;
+    if (!orgId) {
+      throw new Error("No organization found for this account.");
+    }
+    return orgId;
+  } finally {
+    clearTimeout(timeoutId);
   }
-  return orgId;
 }
 
 interface AllauthSessionResponse {
@@ -627,25 +663,37 @@ export async function fetchCurrentUser(
 ): Promise<PlatformUser> {
   const resolvedUrl = platformUrl || getPlatformUrl();
   const url = `${resolvedUrl}/_allauth/app/v1/auth/session`;
-  const response = await fetch(url, {
-    headers: { "X-Session-Token": token },
-  });
 
-  if (!response.ok) {
-    if (
-      response.status === 401 ||
-      response.status === 403 ||
-      response.status === 410
-    ) {
-      throw new Error("Invalid or expired token. Please login again.");
+  const controller = new AbortController();
+  const timeoutId = setTimeout(
+    () => controller.abort(),
+    PLATFORM_FETCH_TIMEOUT_MS,
+  );
+
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: { "X-Session-Token": token },
+    });
+
+    if (!response.ok) {
+      if (
+        response.status === 401 ||
+        response.status === 403 ||
+        response.status === 410
+      ) {
+        throw new Error("Invalid or expired token. Please login again.");
+      }
+      throw new Error(
+        `Platform API error: ${response.status} ${response.statusText}`,
+      );
     }
-    throw new Error(
-      `Platform API error: ${response.status} ${response.statusText}`,
-    );
-  }
 
-  const body = (await response.json()) as AllauthSessionResponse;
-  return body.data.user;
+    const body = (await response.json()) as AllauthSessionResponse;
+    return body.data.user;
+  } finally {
+    clearTimeout(timeoutId);
+  }
 }
 
 // ---------------------------------------------------------------------------

package/src/lib/port-allocator.ts

@@ -0,0 +1,93 @@
+import { createServer } from "net";
+
+/**
+ * Walks upward from `preferred` and returns the first host port that the
+ * kernel will let us bind to. Used by `hatchDocker` to pick the gateway's
+ * host-side port instead of always grabbing the env-default (e.g. 7830 /
+ * 20100), which collides with any other local assistant — eval-spawned or
+ * otherwise — already bound there.
+ *
+ * The previous design (`evals/src/lib/orphan-cleanup.ts`) tried to fix this
+ * by sweeping dead eval-run resources before the next hatch. That only
+ * helped when the conflict came from a prior eval run; an unrelated local
+ * `vellum hatch` holding the port wedged the whole flow. Discovering an
+ * open port at hatch time is the proper fix and lets us delete the cleanup
+ * pre-flight entirely.
+ *
+ * Walks linearly from `preferred` upward rather than asking the kernel for
+ * an arbitrary ephemeral port (`listen(0)`) so the resulting port stays
+ * legible to operators — three local assistants land on N, N+1, N+2
+ * instead of three random numbers in the 32768-60999 range.
+ */
+export async function findOpenPort(
+  preferred: number,
+  options: { maxAttempts?: number; host?: string } = {},
+): Promise<number> {
+  const maxAttempts = options.maxAttempts ?? 50;
+  const host = options.host ?? "0.0.0.0";
+
+  if (!Number.isInteger(preferred) || preferred < 1 || preferred > 65535) {
+    throw new Error(
+      `findOpenPort: preferred port ${preferred} is not a valid TCP port`,
+    );
+  }
+  if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+    throw new Error(
+      `findOpenPort: maxAttempts ${maxAttempts} must be a positive integer`,
+    );
+  }
+
+  let lastError: Error | null = null;
+  for (let offset = 0; offset < maxAttempts; offset++) {
+    const port = preferred + offset;
+    if (port > 65535) break;
+    try {
+      await probePort(port, host);
+      return port;
+    } catch (err) {
+      lastError = err as Error;
+      if (!isPortInUseError(err)) {
+        // EACCES / EPERM / etc. are not "try the next port" signals — those
+        // are configuration problems an operator needs to see immediately.
+        throw err;
+      }
+    }
+  }
+  throw new Error(
+    `findOpenPort: no open port in range [${preferred}, ${preferred + maxAttempts - 1}]` +
+      (lastError ? ` (last error: ${lastError.message})` : ""),
+  );
+}
+
+/**
+ * Resolves if `port` on `host` can be bound right now. Rejects with the
+ * server's `error` event (typically `EADDRINUSE`) otherwise. Always closes
+ * the probe server before resolving so we don't leak the port we just
+ * proved was free.
+ */
+function probePort(port: number, host: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    const cleanup = (cb: () => void): void => {
+      server.removeAllListeners();
+      server.close(() => cb());
+    };
+    server.once("error", (err) => {
+      // close() on a server that never listened is a no-op; calling it
+      // anyway keeps cleanup uniform.
+      cleanup(() => reject(err));
+    });
+    server.once("listening", () => {
+      cleanup(() => resolve());
+    });
+    server.listen(port, host);
+  });
+}
+
+function isPortInUseError(err: unknown): boolean {
+  if (err instanceof Error && "code" in err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    return code === "EADDRINUSE" || code === "EADDRNOTAVAIL";
+  }
+  return false;
+}

package/src/lib/process.ts

@@ -1,6 +1,8 @@
 import { execFileSync } from "child_process";
 import { existsSync, readFileSync, unlinkSync } from "fs";
 
+import { httpHealthCheck, waitForDaemonReady } from "./http-client.js";
+
 /**
  * Verify that a PID belongs to a vellum-related process by inspecting its
  * command line via `ps`. Prevents killing unrelated processes when a PID file
@@ -21,13 +23,15 @@ export function isVellumProcess(pid: number): boolean {
   }
 }
 
+/** Discriminated union: when `alive` is true, `pid` is guaranteed non-null. */
+export type ProcessAliveResult =
+  | { alive: true; pid: number }
+  | { alive: false; pid: null };
+
 /**
  * Check if a PID file's process is alive.
  */
-export function isProcessAlive(pidFile: string): {
-  alive: boolean;
-  pid: number | null;
-} {
+export function isProcessAlive(pidFile: string): ProcessAliveResult {
   if (!existsSync(pidFile)) {
     return { alive: false, pid: null };
   }
@@ -46,6 +50,91 @@ export function isProcessAlive(pidFile: string): {
   }
 }
 
+/** Discriminated union: when `alive` is true, `pid` is guaranteed non-null. */
+export type ProcessHealthResult =
+  | { alive: true; healthy: boolean; pid: number }
+  | { alive: false; healthy: false; pid: null };
+
+/**
+ * Check if a PID file's process is alive AND responding to HTTP health checks.
+ *
+ * Combines PID existence check with an HTTP `/healthz` probe. A process that
+ * exists but does not respond (hung, deadlocked, at 100% CPU) returns
+ * `alive: true, healthy: false` — callers should kill and restart it.
+ */
+export async function isProcessHealthy(
+  pidFile: string,
+  healthPort: number,
+  timeoutMs: number = 3000,
+): Promise<ProcessHealthResult> {
+  const { alive, pid } = isProcessAlive(pidFile);
+  if (!alive || pid === null) {
+    return { alive: false, healthy: false, pid: null };
+  }
+
+  const healthy = await httpHealthCheck(healthPort, timeoutMs);
+  return { alive: true, healthy, pid };
+}
+
+/**
+ * Outcome of {@link resolveProcessState}. Callers switch on `status`:
+ * - `"healthy"` — process is alive and responding; `pid` is the live PID.
+ * - `"needs_start"` — process was dead, hung (and killed), or a stale PID
+ *   was cleaned up. Caller should start a fresh process.
+ */
+export type ProcessState =
+  | { status: "healthy"; pid: number }
+  | { status: "needs_start"; pid: number | null };
+
+/**
+ * Determine whether a PID-tracked process is alive and healthy. If the
+ * process exists but is unresponsive, waits up to `readinessWaitMs`
+ * (default 60s — matches the spawner's own `waitForDaemonReady` timeout
+ * so a concurrent caller never kills a daemon the spawner is still
+ * waiting on) for it to finish initializing. If it remains unresponsive,
+ * verifies it belongs to Vellum before killing it, then cleans up the
+ * PID file.
+ *
+ * Encapsulates the full health → readiness-wait → guard → kill → cleanup
+ * flow so callers don't need to reimplement it.
+ */
+export async function resolveProcessState(
+  pidFile: string,
+  healthPort: number,
+  label: string,
+  readinessWaitMs: number = 60_000,
+): Promise<ProcessState> {
+  const result = await isProcessHealthy(pidFile, healthPort);
+
+  if (!result.alive) {
+    return { status: "needs_start", pid: null };
+  }
+
+  if (result.healthy) {
+    return { status: "healthy", pid: result.pid };
+  }
+
+  // Alive but not healthy — may still be starting up.
+  const becameHealthy = await waitForDaemonReady(healthPort, readinessWaitMs);
+  if (becameHealthy) {
+    return { status: "healthy", pid: result.pid };
+  }
+
+  // Genuinely hung — kill if it belongs to Vellum, otherwise just clean up.
+  if (isVellumProcess(result.pid)) {
+    console.log(
+      `${label} process alive (pid ${result.pid}) but not responding — killing and restarting...`,
+    );
+    await stopProcess(result.pid, label);
+  } else {
+    console.log(
+      `Stale PID file (pid ${result.pid} is not a Vellum process) — cleaning up...`,
+    );
+  }
+  removeFiles(pidFile);
+  return { status: "needs_start", pid: result.pid };
+}
+
 /**
  * Stop a process by PID: SIGTERM, wait up to `timeoutMs`, then SIGKILL if still alive.
  * Returns true if the process was stopped, false if it wasn't alive.
@@ -85,6 +174,18 @@ export async function stopProcess(
   return true;
 }
 
+/** Remove one or more files, ignoring missing-file errors. */
+function removeFiles(...files: (string | string[] | undefined)[]): void {
+  for (const entry of files) {
+    if (!entry) continue;
+    for (const f of Array.isArray(entry) ? entry : [entry]) {
+      try {
+        unlinkSync(f);
+      } catch {}
+    }
+  }
+}
+
 /**
  * Stop a process tracked by a PID file, then clean up the file.
  * Returns true if the process was stopped, false if it wasn't alive.
@@ -92,24 +193,13 @@ export async function stopProcess(
 export async function stopProcessByPidFile(
   pidFile: string,
   label: string,
-  cleanupFiles?: string[],
+  extraCleanupFiles?: string[],
   timeoutMs?: number,
 ): Promise<boolean> {
   const { alive, pid } = isProcessAlive(pidFile);
 
   if (!alive || pid === null) {
-    if (existsSync(pidFile)) {
-      try {
-        unlinkSync(pidFile);
-      } catch {}
-    }
-    if (cleanupFiles) {
-      for (const f of cleanupFiles) {
-        try {
-          unlinkSync(f);
-        } catch {}
-      }
-    }
+    removeFiles(pidFile, extraCleanupFiles);
     return false;
   }
 
@@ -120,32 +210,12 @@ export async function stopProcessByPidFile(
     console.log(
       `PID ${pid} is not a vellum process — cleaning up stale ${label} PID file.`,
     );
-    try {
-      unlinkSync(pidFile);
-    } catch {}
-    if (cleanupFiles) {
-      for (const f of cleanupFiles) {
-        try {
-          unlinkSync(f);
-        } catch {}
-      }
-    }
+    removeFiles(pidFile, extraCleanupFiles);
     return false;
   }
 
   const stopped = await stopProcess(pid, label, timeoutMs);
-
-  try {
-    unlinkSync(pidFile);
-  } catch {}
-  if (cleanupFiles) {
-    for (const f of cleanupFiles) {
-      try {
-        unlinkSync(f);
-      } catch {}
-    }
-  }
-
+  removeFiles(pidFile, extraCleanupFiles);
   return stopped;
 }
 

package/src/lib/statefulset.ts

@@ -257,7 +257,6 @@ export interface BuildServiceRunArgsOpts extends DockerRunSecrets {
   instanceName: string;
   res: DockerResourceNames;
   extraAssistantEnv?: Record<string, string>;
-  defaultWorkspaceConfigPath?: string;
   /** Avatar device path, if available. Injected by `docker.ts` after resolving. */
   avatarDevicePath?: string;
 }
@@ -286,7 +285,6 @@ export function buildServiceRunArgs(
     instanceName,
     res,
     extraAssistantEnv,
-    defaultWorkspaceConfigPath,
     avatarDevicePath,
   } = opts;
 
@@ -355,14 +353,6 @@ export function buildServiceRunArgs(
           "-e", `GATEWAY_INTERNAL_URL=http://localhost:${GATEWAY_INTERNAL_PORT}`,
         );
 
-        if (defaultWorkspaceConfigPath) {
-          const cPath = `/tmp/vellum-default-workspace-config-${Date.now()}.json`;
-          args.push(
-            "-v", `${defaultWorkspaceConfigPath}:${cPath}:ro`,
-            "-e", `VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH=${cPath}`,
-          );
-        }
-
         if (extraAssistantEnv) {
           for (const [k, v] of Object.entries(extraAssistantEnv)) {
             args.push("-e", `${k}=${v}`);

package/src/lib/step-runner.ts

@@ -1,5 +1,38 @@
 import { spawn } from "child_process";
 
+/**
+ * Build the error message for a failed child process. **Never include the
+ * argv** — `docker run ...` invocations carry `-e ANTHROPIC_API_KEY=…` /
+ * `-e OPENAI_API_KEY=…` style flags, and the resulting `Error.message`
+ * propagates all the way to:
+ *
+ *   - the CLI's top-level catch (`console.error("Error:", err.message)`)
+ *     which leaks them onto stderr,
+ *   - `subprocess-*.log` files captured by the evals harness when it
+ *     spawns `vellum hatch` (which then becomes the inlined log on the
+ *     run-detail report page),
+ *   - `run.json#error` and the last-N-lines tail in `progress.ndjson`
+ *     that the evals harness emits for `SubprocessFailedError`.
+ *
+ * The diagnostic substring callers actually grep for ("no such container",
+ * "is not running", "port is already allocated", …) lives in the child's
+ * stderr/stdout, which we DO preserve below. Keep the command name only —
+ * it's enough to disambiguate which step failed without quoting secrets.
+ *
+ * Exported so the unit test can assert no `-e KEY=...` slips back in.
+ */
+export function buildExecErrorMessage(
+  command: string,
+  code: number | null,
+  stderr: string,
+  stdout: string,
+): string {
+  const codeLabel = code === null ? "an unknown code" : `code ${code}`;
+  const header = `${command} exited with ${codeLabel}`;
+  const output = [stderr.trim(), stdout.trim()].filter(Boolean).join("\n");
+  return output ? `${header}\n${output}` : header;
+}
+
 export function exec(
   command: string,
   args: string[],
@@ -25,21 +58,59 @@ export function exec(
       if (code === 0) {
         resolve();
       } else {
-        const msg = `"${command} ${args.join(" ")}" exited with code ${code}`;
-        const output = [stderr.trim(), stdout.trim()]
-          .filter(Boolean)
-          .join("\n");
-        reject(new Error(output ? `${msg}\n${output}` : msg));
+        reject(new Error(buildExecErrorMessage(command, code, stderr, stdout)));
       }
     });
     child.on("error", reject);
   });
 }
 
-export function execOutput(
+/**
+ * Run `command` with `args` and pipe `input` to its stdin. Mirrors `exec` —
+ * same no-args-in-error-message contract from `buildExecErrorMessage` — but
+ * lets callers stream content (e.g. a small JSON blob) into a child process
+ * without having to put the content on the command line where `ps` could
+ * read it and where Docker bind-mounts would be involved.
+ */
+export function execWithStdin(
   command: string,
   args: string[],
+  input: string,
   options: { cwd?: string } = {},
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd,
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    child.stdout.on("data", (data: Buffer) => {
+      stdout += data.toString();
+    });
+
+    let stderr = "";
+    child.stderr.on("data", (data: Buffer) => {
+      stderr += data.toString();
+    });
+
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(buildExecErrorMessage(command, code, stderr, stdout)));
+      }
+    });
+    child.on("error", reject);
+
+    child.stdin.end(input);
+  });
+}
+
+export function execOutput(
+  command: string,
+  args: string[],
+  options: { cwd?: string; timeoutMs?: number } = {},
 ): Promise<string> {
   return new Promise((resolve, reject) => {
     const child = spawn(command, args, {
@@ -47,6 +118,21 @@ export function execOutput(
       stdio: ["pipe", "pipe", "pipe"],
     });
 
+    let settled = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+
+    if (options.timeoutMs !== undefined) {
+      timer = setTimeout(() => {
+        if (!settled) {
+          settled = true;
+          child.kill("SIGTERM");
+          reject(
+            new Error(`${command} timed out after ${options.timeoutMs}ms`),
+          );
+        }
+      }, options.timeoutMs);
+    }
+
     let stdout = "";
     child.stdout.on("data", (data: Buffer) => {
       stdout += data.toString();
@@ -58,13 +144,20 @@ export function execOutput(
     });
 
     child.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
       if (code === 0) {
         resolve(stdout.trim());
       } else {
-        const msg = `"${command} ${args.join(" ")}" exited with code ${code}`;
-        reject(new Error(stderr.trim() ? `${msg}\n${stderr.trim()}` : msg));
+        reject(new Error(buildExecErrorMessage(command, code, stderr, "")));
       }
     });
-    child.on("error", reject);
+    child.on("error", (err) => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      reject(err);
+    });
   });
 }