@vm0/runner 3.12.3 → 3.14.0

package/index.js

@@ -429,6 +429,26 @@ function isProcessRunning(pid) {
     return false;
   }
 }
+async function gracefulKillProcess(pid, timeoutMs = 5e3) {
+  if (!isProcessRunning(pid)) return true;
+  try {
+    process.kill(pid, "SIGTERM");
+  } catch {
+    return !isProcessRunning(pid);
+  }
+  const startTime = Date.now();
+  while (Date.now() - startTime < timeoutMs) {
+    if (!isProcessRunning(pid)) return true;
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  if (isProcessRunning(pid)) {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+    }
+  }
+  return !isProcessRunning(pid);
+}
 function killProcessTree(pid) {
   try {
     const childPidsStr = execSync(`pgrep -P ${pid} 2>/dev/null || true`, {
@@ -9935,7 +9955,7 @@ async function executeJob(context, config, options = {}) {
     logger9.log(`Guest client ready`);
     if (config.firecracker.snapshot) {
       const timestamp = (Date.now() / 1e3).toFixed(3);
-      await guest.exec(`date -s "@${timestamp}"`);
+      await guest.exec(`sudo date -s "@${timestamp}"`);
     }
     if (context.storageManifest) {
       await withSandboxTiming(
@@ -9994,7 +10014,7 @@ async function executeJob(context, config, options = {}) {
     const duration = Math.round(durationMs / 1e3);
     if (exitCode === 137 || exitCode === 9) {
       const dmesgCheck = await guest.exec(
-        `dmesg | tail -20 | grep -iE "killed|oom" 2>/dev/null`
+        `sudo dmesg | tail -20 | grep -iE "killed|oom" 2>/dev/null`
       );
       if (dmesgCheck.stdout.toLowerCase().includes("oom") || dmesgCheck.stdout.toLowerCase().includes("killed")) {
         logger9.log(`OOM detected: ${dmesgCheck.stdout}`);
@@ -10519,9 +10539,20 @@ import { Command as Command2 } from "commander";
 import { existsSync as existsSync5, readFileSync as readFileSync3, readdirSync as readdirSync2 } from "fs";
 import { execSync as execSync3 } from "child_process";
 
-// src/lib/firecracker/process.ts
-import { readdirSync, readFileSync as readFileSync2, existsSync as existsSync4 } from "fs";
+// src/lib/process.ts
+import { readdirSync, readFileSync as readFileSync2, readlinkSync, existsSync as existsSync4 } from "fs";
 import path8 from "path";
+function isOrphanProcess(pid) {
+  try {
+    const stat = readFileSync2(`/proc/${pid}/stat`, "utf-8");
+    const lastParen = stat.lastIndexOf(")");
+    if (lastParen === -1) return false;
+    const fields = stat.slice(lastParen + 1).trim().split(/\s+/);
+    return fields[1] === "1";
+  } catch {
+    return false;
+  }
+}
 function parseFirecrackerCmdline(cmdline) {
   const args = cmdline.split("\0");
   if (!args[0]?.includes("firecracker")) return null;
@@ -10556,6 +10587,27 @@ function parseMitmproxyCmdline(cmdline) {
   }
   return null;
 }
+function parseRunnerCmdline(cmdline) {
+  const args = cmdline.split("\0").filter((a) => a !== "");
+  const startIdx = args.indexOf("start");
+  const benchmarkIdx = args.indexOf("benchmark");
+  let mode;
+  let modeIdx;
+  if (startIdx !== -1 && (benchmarkIdx === -1 || startIdx < benchmarkIdx)) {
+    mode = "start";
+    modeIdx = startIdx;
+  } else if (benchmarkIdx !== -1) {
+    mode = "benchmark";
+    modeIdx = benchmarkIdx;
+  } else {
+    return null;
+  }
+  const configIdx = args.indexOf("--config", modeIdx + 1);
+  if (configIdx === -1 || configIdx >= args.length - 1) return null;
+  const configPath = args[configIdx + 1];
+  if (!configPath?.match(/\.ya?ml$/)) return null;
+  return { configPath, mode };
+}
 function findFirecrackerProcesses() {
   const processes = [];
   const procDir = "/proc";
@@ -10574,7 +10626,12 @@ function findFirecrackerProcesses() {
       const cmdline = readFileSync2(cmdlinePath, "utf-8");
       const parsed = parseFirecrackerCmdline(cmdline);
       if (parsed) {
-        processes.push({ pid, vmId: parsed.vmId, baseDir: parsed.baseDir });
+        processes.push({
+          pid,
+          vmId: parsed.vmId,
+          baseDir: parsed.baseDir,
+          isOrphan: isOrphanProcess(pid)
+        });
       }
     } catch {
       continue;
@@ -10587,27 +10644,39 @@ function findProcessByVmId(vmId) {
   const vmIdStr = vmIdValue(vmId);
   return processes.find((p) => vmIdValue(p.vmId) === vmIdStr) || null;
 }
-async function killProcess(pid, timeoutMs = 5e3) {
-  if (!isProcessRunning(pid)) return true;
+function findMitmproxyProcesses() {
+  const processes = [];
+  const procDir = "/proc";
+  let entries;
   try {
-    process.kill(pid, "SIGTERM");
+    entries = readdirSync(procDir);
   } catch {
-    return !isProcessRunning(pid);
-  }
-  const startTime = Date.now();
-  while (Date.now() - startTime < timeoutMs) {
-    if (!isProcessRunning(pid)) return true;
-    await new Promise((resolve) => setTimeout(resolve, 100));
+    return [];
   }
-  if (isProcessRunning(pid)) {
+  for (const entry of entries) {
+    if (!/^\d+$/.test(entry)) continue;
+    const pid = parseInt(entry, 10);
+    const cmdlinePath = path8.join(procDir, entry, "cmdline");
+    if (!existsSync4(cmdlinePath)) continue;
     try {
-      process.kill(pid, "SIGKILL");
+      const cmdline = readFileSync2(cmdlinePath, "utf-8");
+      const baseDir = parseMitmproxyCmdline(cmdline);
+      if (baseDir) {
+        processes.push({ pid, baseDir, isOrphan: isOrphanProcess(pid) });
+      }
     } catch {
+      continue;
     }
   }
-  return !isProcessRunning(pid);
+  return processes;
 }
-function findMitmproxyProcesses() {
+function isNodeIndexJs(cmdline) {
+  const args = cmdline.split("\0").filter((a) => a !== "");
+  if (args.length < 2) return false;
+  if (!args[0]?.includes("node")) return false;
+  return args[1]?.endsWith("index.js") ?? false;
+}
+function findRunnerProcesses() {
   const processes = [];
   const procDir = "/proc";
   let entries;
@@ -10623,9 +10692,27 @@ function findMitmproxyProcesses() {
     if (!existsSync4(cmdlinePath)) continue;
     try {
       const cmdline = readFileSync2(cmdlinePath, "utf-8");
-      const baseDir = parseMitmproxyCmdline(cmdline);
-      if (baseDir) {
-        processes.push({ pid, baseDir });
+      const parsed = parseRunnerCmdline(cmdline);
+      if (parsed) {
+        processes.push({
+          pid,
+          configPath: parsed.configPath,
+          mode: parsed.mode
+        });
+        continue;
+      }
+      if (isNodeIndexJs(cmdline)) {
+        const cwdPath = path8.join(procDir, entry, "cwd");
+        const cwd = readlinkSync(cwdPath);
+        const configPath = path8.join(cwd, "runner.yaml");
+        if (existsSync4(configPath)) {
+          processes.push({
+            pid,
+            configPath,
+            mode: "start"
+            // Default to start mode (cannot determine from cmdline)
+          });
+        }
       }
     } catch {
       continue;
@@ -10646,68 +10733,39 @@ var RunnerStatusSchema = z31.object({
 });
 
 // src/commands/doctor.ts
-function displayRunnerStatus(statusFilePath, warnings) {
+function getRunnerStatus(statusFilePath, warnings) {
   if (!existsSync5(statusFilePath)) {
-    console.log("Mode: unknown (no status.json)");
     return null;
   }
   try {
-    const status = RunnerStatusSchema.parse(
+    return RunnerStatusSchema.parse(
       JSON.parse(readFileSync3(statusFilePath, "utf-8"))
     );
-    console.log(`Mode: ${status.mode}`);
-    if (status.started_at) {
-      const started = new Date(status.started_at);
-      const uptime = formatUptime(Date.now() - started.getTime());
-      console.log(`Started: ${started.toLocaleString()} (uptime: ${uptime})`);
-    }
-    return status;
   } catch {
-    console.log("Mode: unknown (status.json unreadable)");
     warnings.push({ message: "status.json exists but cannot be parsed" });
     return null;
   }
 }
 async function checkApiConnectivity(config, warnings) {
-  console.log("API Connectivity:");
   try {
     await pollForJob(config.server, config.group);
-    console.log(`  \u2713 Connected to ${config.server.url}`);
-    console.log("  \u2713 Authentication: OK");
+    return true;
   } catch (error) {
-    console.log(`  \u2717 Cannot connect to ${config.server.url}`);
-    console.log(
-      `    Error: ${error instanceof Error ? error.message : "Unknown error"}`
-    );
     warnings.push({
       message: `Cannot connect to API: ${error instanceof Error ? error.message : "Unknown error"}`
     });
+    return false;
   }
 }
-function checkNetwork(config, warnings) {
-  console.log("Network:");
-  const mitmProcesses = findMitmproxyProcesses();
-  const mitmProc = mitmProcesses.find((p) => p.baseDir === config.base_dir);
-  if (mitmProc) {
-    console.log(
-      `  \u2713 Proxy mitmproxy (PID ${mitmProc.pid}) on :${config.proxy.port}`
-    );
-  } else {
-    console.log(`  \u2717 Proxy mitmproxy not running`);
-    warnings.push({ message: "Proxy mitmproxy is not running" });
-  }
-  console.log(
-    `  \u2139 Namespaces: each VM runs in isolated namespace with IP ${SNAPSHOT_NETWORK.guestIp}`
-  );
-}
-function buildJobInfo(status, processes) {
+function buildJobInfo(status, processes, baseDir) {
   const jobs = [];
   const statusVmIds = /* @__PURE__ */ new Set();
+  const runnerProcesses = processes.filter((p) => p.baseDir === baseDir);
   if (status?.active_run_ids) {
     for (const runId of status.active_run_ids) {
       const vmId = createVmId(runId);
       statusVmIds.add(vmId);
-      const proc = processes.find((p) => p.vmId === vmId);
+      const proc = runnerProcesses.find((p) => p.vmId === vmId);
       jobs.push({
         runId,
         vmId,
@@ -10717,18 +10775,6 @@ function buildJobInfo(status, processes) {
   }
   return { jobs, statusVmIds };
 }
-function displayRuns(jobs, maxConcurrent) {
-  console.log(`Runs (${jobs.length} active, max ${maxConcurrent}):`);
-  if (jobs.length === 0) {
-    console.log("  No active runs");
-    return;
-  }
-  console.log("  Run ID                                VM ID       Status");
-  for (const job of jobs) {
-    const statusText = job.firecrackerPid ? `\u2713 Running (PID ${job.firecrackerPid})` : "\u26A0\uFE0F No process";
-    console.log(`  ${job.runId}  ${job.vmId}    ${statusText}`);
-  }
-}
 async function findOrphanNetworkNamespaces(warnings) {
   let allNamespaces = [];
   try {
@@ -10777,12 +10823,12 @@ async function findOrphanNetworkNamespaces(warnings) {
     return [];
   }
 }
-async function detectOrphanResources(jobs, allProcesses, workspaces, statusVmIds, baseDir, warnings) {
+function detectRunnerOrphanResources(jobs, allProcesses, workspaces, statusVmIds, baseDir, warnings) {
   const processes = allProcesses.filter((p) => p.baseDir === baseDir);
   for (const job of jobs) {
     if (!job.firecrackerPid) {
       warnings.push({
-        message: `Run ${job.vmId} in status.json but no Firecracker process running`
+        message: `Run ${job.vmId} in status.json but no Firecracker process`
       });
     }
   }
@@ -10794,31 +10840,15 @@ async function detectOrphanResources(jobs, allProcesses, workspaces, statusVmIds
       });
     }
   }
-  const orphanNetns = await findOrphanNetworkNamespaces(warnings);
-  for (const ns of orphanNetns) {
-    warnings.push({
-      message: `Orphan network namespace: ${ns} (runner process not running)`
-    });
-  }
   for (const ws of workspaces) {
     const vmId = runnerPaths.extractVmId(ws);
     if (!processVmIds.has(vmId) && !statusVmIds.has(vmId)) {
       warnings.push({
-        message: `Orphan workspace: ${ws} (no matching job or process)`
+        message: `Orphan workspace: ${ws}`
       });
     }
   }
 }
-function displayWarnings(warnings) {
-  console.log("Warnings:");
-  if (warnings.length === 0) {
-    console.log("  None");
-  } else {
-    for (const w of warnings) {
-      console.log(`  - ${w.message}`);
-    }
-  }
-}
 function formatUptime(ms) {
   const seconds = Math.floor(ms / 1e3);
   const minutes = Math.floor(seconds / 60);
@@ -10829,34 +10859,165 @@ function formatUptime(ms) {
   if (minutes > 0) return `${minutes}m`;
   return `${seconds}s`;
 }
-var doctorCommand = new Command2("doctor").description("Diagnose runner health, check network, and detect issues").option("--config <path>", "Config file path", "./runner.yaml").action(async (options) => {
+async function displayRunnerHealth(runner, index, allFirecrackerProcesses, allMitmproxyProcesses) {
+  const warnings = [];
+  const { config, pid, mode } = runner;
+  const baseDir = config.base_dir;
+  console.log(`[${index}] ${baseDir} (PID ${pid}) [${mode}]`);
+  const statusFilePath = runnerPaths.statusFile(baseDir);
+  const status = getRunnerStatus(statusFilePath, warnings);
+  if (status) {
+    let statusLine = `    Mode: ${status.mode}`;
+    if (status.started_at) {
+      const started = new Date(status.started_at);
+      const uptime = formatUptime(Date.now() - started.getTime());
+      statusLine += `, uptime: ${uptime}`;
+    }
+    console.log(statusLine);
+  } else {
+    console.log("    Mode: unknown (no status.json)");
+  }
+  const apiOk = await checkApiConnectivity(config, warnings);
+  if (apiOk) {
+    console.log(`    API: \u2713 Connected to ${config.server.url}`);
+  } else {
+    console.log(`    API: \u2717 Cannot connect to ${config.server.url}`);
+  }
+  const mitmProc = allMitmproxyProcesses.find((p) => p.baseDir === baseDir);
+  if (mitmProc) {
+    console.log(
+      `    Proxy: \u2713 mitmproxy (PID ${mitmProc.pid}) on :${config.proxy.port}`
+    );
+  } else if (mode === "start") {
+    console.log("    Proxy: \u2717 not running");
+    warnings.push({ message: "Proxy mitmproxy is not running" });
+  } else {
+    console.log("    Proxy: - (not running)");
+  }
+  const { jobs, statusVmIds } = buildJobInfo(
+    status,
+    allFirecrackerProcesses,
+    baseDir
+  );
+  console.log(
+    `    Runs (${jobs.length} active, max ${config.sandbox.max_concurrent}):`
+  );
+  if (jobs.length === 0) {
+    console.log("      No active runs");
+  } else {
+    for (const job of jobs) {
+      const statusText = job.firecrackerPid ? `\u2713 Running (PID ${job.firecrackerPid})` : "\u26A0\uFE0F No process";
+      console.log(`      ${job.vmId}  ${statusText}`);
+    }
+  }
+  const workspacesDir = runnerPaths.workspacesDir(baseDir);
+  const workspaces = existsSync5(workspacesDir) ? readdirSync2(workspacesDir).filter(runnerPaths.isVmWorkspace) : [];
+  detectRunnerOrphanResources(
+    jobs,
+    allFirecrackerProcesses,
+    workspaces,
+    statusVmIds,
+    baseDir,
+    warnings
+  );
+  console.log(`    Warnings:`);
+  if (warnings.length === 0) {
+    console.log("      None");
+  } else {
+    for (const w of warnings) {
+      console.log(`      - ${w.message}`);
+    }
+  }
+  return warnings;
+}
+async function detectGlobalOrphans(discoveredRunners, allFirecrackerProcesses, allMitmproxyProcesses, globalWarnings) {
+  const runnerBaseDirs = new Set(
+    discoveredRunners.map((r) => r.config.base_dir)
+  );
+  for (const mitm of allMitmproxyProcesses) {
+    if (mitm.isOrphan) {
+      globalWarnings.push({
+        message: `Orphan mitmproxy: PID ${mitm.pid} (PPID=1, parent process dead)`
+      });
+    } else if (!runnerBaseDirs.has(mitm.baseDir)) {
+      globalWarnings.push({
+        message: `Orphan mitmproxy: PID ${mitm.pid} (baseDir ${mitm.baseDir}, runner not running)`
+      });
+    }
+  }
+  for (const fc of allFirecrackerProcesses) {
+    if (fc.isOrphan) {
+      globalWarnings.push({
+        message: `Orphan Firecracker: PID ${fc.pid} (vmId ${fc.vmId}, PPID=1, parent process dead)`
+      });
+    } else if (!runnerBaseDirs.has(fc.baseDir)) {
+      globalWarnings.push({
+        message: `Orphan Firecracker: PID ${fc.pid} (vmId ${fc.vmId}, baseDir ${fc.baseDir}, runner not running)`
+      });
+    }
+  }
+  const orphanNetns = await findOrphanNetworkNamespaces(globalWarnings);
+  for (const ns of orphanNetns) {
+    globalWarnings.push({
+      message: `Orphan namespace: ${ns} (runner process not running)`
+    });
+  }
+}
+var doctorCommand = new Command2("doctor").description("Diagnose health of all runners on this host").action(async () => {
   try {
-    const config = loadConfig(options.config);
-    const statusFilePath = runnerPaths.statusFile(config.base_dir);
-    const workspacesDir = runnerPaths.workspacesDir(config.base_dir);
-    const warnings = [];
-    console.log(`Runner: ${config.name}`);
-    const status = displayRunnerStatus(statusFilePath, warnings);
-    console.log("");
-    await checkApiConnectivity(config, warnings);
-    console.log("");
-    checkNetwork(config, warnings);
-    console.log("");
-    const processes = findFirecrackerProcesses();
-    const workspaces = existsSync5(workspacesDir) ? readdirSync2(workspacesDir).filter(runnerPaths.isVmWorkspace) : [];
-    const { jobs, statusVmIds } = buildJobInfo(status, processes);
-    displayRuns(jobs, config.sandbox.max_concurrent);
+    const globalWarnings = [];
+    let totalWarnings = 0;
+    const allFirecrackerProcesses = findFirecrackerProcesses();
+    const allMitmproxyProcesses = findMitmproxyProcesses();
+    const runnerProcesses = findRunnerProcesses();
+    const discoveredRunners = [];
+    for (const rp of runnerProcesses) {
+      try {
+        const config = loadConfig(rp.configPath);
+        discoveredRunners.push({
+          pid: rp.pid,
+          config,
+          mode: rp.mode
+        });
+      } catch (err) {
+        globalWarnings.push({
+          message: `Failed to load config ${rp.configPath}: ${err instanceof Error ? err.message : "Unknown error"}`
+        });
+      }
+    }
+    console.log(`Runners (${discoveredRunners.length} found):`);
     console.log("");
-    await detectOrphanResources(
-      jobs,
-      processes,
-      workspaces,
-      statusVmIds,
-      config.base_dir,
-      warnings
+    if (discoveredRunners.length === 0) {
+      console.log("    No runner processes found");
+      console.log("");
+    } else {
+      for (let i = 0; i < discoveredRunners.length; i++) {
+        const warnings = await displayRunnerHealth(
+          discoveredRunners[i],
+          i + 1,
+          allFirecrackerProcesses,
+          allMitmproxyProcesses
+        );
+        totalWarnings += warnings.length;
+        console.log("");
+      }
+    }
+    await detectGlobalOrphans(
+      discoveredRunners,
+      allFirecrackerProcesses,
+      allMitmproxyProcesses,
+      globalWarnings
     );
-    displayWarnings(warnings);
-    process.exit(warnings.length > 0 ? 1 : 0);
+    console.log("Global:");
+    if (globalWarnings.length === 0) {
+      console.log("    No orphan resources");
+    } else {
+      for (const w of globalWarnings) {
+        console.log(`    ${w.message}`);
+      }
+    }
+    totalWarnings += globalWarnings.length;
+    process.exit(totalWarnings > 0 ? 1 : 0);
   } catch (error) {
     console.error(
       `Error: ${error instanceof Error ? error.message : "Unknown error"}`
@@ -10900,7 +11061,7 @@ var killCommand = new Command3("kill").description("Force terminate a run and cl
       }
       const results = [];
       if (proc) {
-        const killed = await killProcess(proc.pid);
+        const killed = await gracefulKillProcess(proc.pid);
         results.push({
           step: "Firecracker process",
           success: killed,
@@ -11345,7 +11506,7 @@ var snapshotCommand = new Command5("snapshot").description("Generate a Firecrack
 );
 
 // src/index.ts
-var version = true ? "3.12.3" : "0.1.0";
+var version = true ? "3.14.0" : "0.1.0";
 program.name("vm0-runner").version(version).description("Self-hosted runner for VM0 agents");
 program.addCommand(startCommand);
 program.addCommand(doctorCommand);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vm0/runner",
-  "version": "3.12.3",
+  "version": "3.14.0",
   "description": "Self-hosted runner for VM0 agents",
   "repository": {
     "type": "git",