@runuai/host 0.9.158 → 0.9.160

package/lib/orchestrator.ts

@@ -9374,10 +9374,54 @@ async function recoverPersistedTaskEnvironment(
     return result.outcome === "recovered";
   } catch (error) {
     if (tunnelReadinessFailed) return false;
+    if (error instanceof TaskRecoveryRowConflictError && isCurrent()) {
+      return quarantineConflictingTask(task, environment, error, isCurrent, lifecycle);
+    }
     throw error;
   }
 }
 
+/** A durable row the provider can never reconcile with its locator. Retrying
+ * cannot change the verdict, so the sweep must not treat it as an unknown
+ * runtime state: that kept the whole runtime in checking, retrying every
+ * ~50 s, while one task's row was wrong (2026-10-19). */
+export class TaskRecoveryRowConflictError extends Error {
+  override name = "TaskRecoveryRowConflictError";
+}
+
+/** Settle one task whose row conflicts with its locator without holding the
+ * runtime: drain its consumers, stop the locator's environment, and mark the
+ * row `error` so the task reads as failed rather than running. Only a stop
+ * that cannot be proven throws on, keeping the sweep unresolved, because
+ * then the runtime state really is unknown. Resume or Destroy clears it. */
+async function quarantineConflictingTask(
+  task: typeof schema.hostTasks.$inferSelect,
+  environment: TaskEnvironmentHandle<unknown>,
+  conflict: TaskRecoveryRowConflictError,
+  isCurrent: () => boolean,
+  lifecycle?: Orchestrator,
+): Promise<boolean> {
+  await lifecycle?.quarantineChannelForRecovery(task.taskId);
+  if (!isCurrent()) return true;
+  await environment.stop();
+  const after = await environment.status();
+  if (!isCurrent()) return true;
+  if (after.state === "running" || after.state === "unknown") {
+    throw new Error(
+      `${conflict.message}, and its environment could not be proven stopped`,
+    );
+  }
+  db_setStatus(task.taskId, "error", {
+    codeServerPort: null,
+    previewPorts: "[]",
+  });
+  clearTaskGatewayAcl(task.taskId);
+  console.error(
+    `[orchestrator] recovery: ${task.taskId} quarantined: ${conflict.message}; its environment was stopped and the task marked error so the runtime can go operational`,
+  );
+  return true;
+}
+
 interface RecoveryTaskConsumers {
   quarantineChannelForRecovery(taskId: string): Promise<void>;
   prepareChannelForRecovery(taskId: string): Promise<void>;
@@ -10020,23 +10064,28 @@ async function recoverDockerTaskEnvironment(
   if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
     return { outcome: "recovered" };
   }
-  const isPreparedRow =
-    (task.statusMirror === "starting" || task.statusMirror === "error") &&
-    task.composeProject === null &&
-    task.worktreePath === null;
+  // A starting/error row may hold only part of its legacy mirrors: a crash
+  // before task-up finished leaves both NULL, and a start that failed after
+  // an earlier recovery settled the row half-written leaves compose_project
+  // set with worktree_path NULL (live 2026-10-19: one such row held the
+  // whole runtime in checking for 2.5 h). Each mirror present must still
+  // equal the locator; a missing one is adopted from it.
+  const mayAdoptLocator =
+    task.statusMirror === "starting" || task.statusMirror === "error";
+  const mirrorAgrees = (mirror: string | null, expected: string): boolean =>
+    mirror === expected || (mayAdoptLocator && mirror === null);
   if (
-    !isPreparedRow &&
-    (task.composeProject !== locator.composeProject ||
-      task.worktreePath !== locator.hostWorktreePath)
+    !mirrorAgrees(task.composeProject, locator.composeProject) ||
+    !mirrorAgrees(task.worktreePath, locator.hostWorktreePath)
   ) {
-    throw new Error(
+    throw new TaskRecoveryRowConflictError(
       `task ${descriptor.taskId} environment locator no longer matches its durable row`,
     );
   }
   // Provider-owned provisional metadata is authoritative after a crash. Feed
   // the already-validated deterministic values to the Docker-private repair
   // helpers without teaching the generic sweep how to derive them.
-  const recoveryTask = isPreparedRow
+  const recoveryTask = mayAdoptLocator
     ? {
         ...task,
         composeProject: locator.composeProject,
@@ -10147,7 +10196,13 @@ async function recoverOneDockerTask(
     db_setStatus(task.taskId, next, {
       codeServerPort: null,
       previewPorts: "[]",
-      composeProject: next === "error" ? null : composeProject,
+      // A prepared row reaches here with both mirrors adopted from its
+      // locator: write them back as a pair. Writing compose_project alone
+      // left a stopped row whose next failed Resume became the half-written
+      // error row of 2026-10-19.
+      ...(next === "error"
+        ? { composeProject: null }
+        : { composeProject, worktreePath: task.worktreePath }),
     });
     console.log(
       `[orchestrator] recovery: ${task.taskId} -> ${next} (container gone, ` +

package/lib/task-files-machine-retirement-outbox.ts

@@ -243,6 +243,77 @@ export function taskMachinePeerProcessIds(): number[] {
     );
 }
 
+/** A process's parent from /proc; null once the process is gone. */
+export function processParentId(pid: number): number | null {
+  let stat: string;
+  try {
+    stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+  // `pid (comm) state ppid ...`: comm may itself hold spaces or parens.
+  const ppid = Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[1]);
+  if (!Number.isSafeInteger(ppid) || ppid < 0) {
+    throw new Error(`process ${pid} has an unreadable parent`);
+  }
+  return ppid;
+}
+
+/**
+ * Every process in the sidecar service cgroup that this runner did not start.
+ * The fresh-incarnation proof asks this, not `taskMachinePeerProcessIds`: the
+ * runner's own helpers share its cgroup from boot, tsx's esbuild service above
+ * all (a long-lived child of every runner that compiled a module; live
+ * 2026-10-20, every marked machine stayed pending "not quiescent" on it), and
+ * a pending runner admits no task, so nothing it started is a task process.
+ * What an earlier runner process left behind (KillMode=process) was reparented
+ * when that process ended: it is never this runner's descendant, so it counts.
+ */
+export function taskMachineForeignProcessIds(): number[] {
+  return foreignProcessIdsWith({
+    list: taskMachinePeerProcessIds,
+    parent: processParentId,
+    self: process.pid,
+  });
+}
+
+export interface ForeignProcessDeps {
+  list(): readonly number[];
+  parent(pid: number): number | null;
+  self: number;
+}
+
+/** Exported seam keeps the ancestry walk executable in tests. */
+export function foreignProcessIdsWith(deps: ForeignProcessDeps): number[] {
+  const foreign: number[] = [];
+  for (const pid of deps.list()) {
+    if (pid === deps.self) continue;
+    const seen = new Set<number>();
+    let current: number | null = pid;
+    let ancestry: "ours" | "foreign" | "gone" = "foreign";
+    for (;;) {
+      const parent = deps.parent(current);
+      if (parent === null) {
+        // The process itself is gone: nothing to count. An ancestor gone
+        // mid-walk leaves an orphan the next look sees reparented: fail
+        // closed now, as that look would.
+        if (current === pid) ancestry = "gone";
+        break;
+      }
+      if (parent === deps.self) {
+        ancestry = "ours";
+        break;
+      }
+      if (parent <= 1 || seen.has(parent)) break;
+      seen.add(parent);
+      current = parent;
+    }
+    if (ancestry === "foreign") foreign.push(pid);
+  }
+  return foreign;
+}
+
 /** Kill everything in the sidecar service cgroup except this root runner.
  * The task's agents, editor, terminals and sudo children all descend from the
  * service; KillMode=process is precisely why they otherwise survive. */

package/lib/task-machine-credentials-activation.ts

@@ -26,6 +26,8 @@ export interface TaskMachineCredentialsActivationDeps {
   filesReady(): boolean;
   retirementReady(): boolean;
   taskRows(): readonly { taskId: string }[];
+  /** Processes in the sidecar's service cgroup that this runner did not
+   *  start (its own helpers, such as tsx's esbuild service, never count). */
   peerProcessIds(): readonly number[];
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.9.158",
+  "version": "0.9.160",
   "description": "Uai host — runs ephemeral AI tasks in containers on a machine you control.",
   "license": "MIT",
   "author": "Uai Tech <team@runuai.com>",

package/src/main.ts

@@ -428,7 +428,7 @@ import {
   kernelBootId,
   TaskFilesMachineRetirementOutbox,
   TASK_FILES_MACHINE_RETIREMENT_RECORD,
-  taskMachinePeerProcessIds,
+  taskMachineForeignProcessIds,
 } from "../lib/task-files-machine-retirement-outbox";
 import {
   planFreshTaskMachineCredentialsActivation,
@@ -1819,7 +1819,8 @@ function taskMachineCredentialsActivationDeps(): TaskMachineCredentialsActivatio
         .select({ taskId: schema.hostTasks.taskId })
         .from(schema.hostTasks)
         .all(),
-    peerProcessIds: taskMachinePeerProcessIds,
+    // Not the runner's own children (tsx's esbuild service): see the helper.
+    peerProcessIds: taskMachineForeignProcessIds,
   };
 }