@runuai/host 0.8.17 → 0.8.19

package/lib/agents/transport.ts

@@ -66,6 +66,25 @@ export interface AgentTransportOptions {
 /** Runner heartbeat is 5 s; older than this = not attachable. */
 const ATTACH_HEARTBEAT_FRESH_MS = 20_000;
 
+/**
+ * ONE live outbox consumer per (task, agent) in this host process. A session
+ * dropped from the orchestrator's map without close()/detach() (e.g. the
+ * error-event path — a pipes-era "process is dead" assumption) leaves its
+ * DurableProcess polling FOREVER: its liveness check only trips when the
+ * RUNNER dies, and the runner is alive. The next channelEnsure then attaches
+ * a second tail to the same outbox, and every agent turn is emitted once per
+ * leaked tail — live incident 2026-07-20: one agent's messages posted ×5 to
+ * the feed, multiplicity growing by one per errored turn. Enforcing the
+ * invariant here covers every replacement path, including future ones.
+ */
+const liveTails = new Map<string, DurableProcess>();
+
+function claimTail(key: string, proc: DurableProcess): DurableProcess {
+  liveTails.get(key)?.detach();
+  liveTails.set(key, proc);
+  return proc;
+}
+
 function durableEnabled(): boolean {
   return process.env.UAI_DURABLE_SESSIONS !== "0";
 }
@@ -117,6 +136,8 @@ export function createAgentTransport(opts: AgentTransportOptions): LineTransport
       .run();
   };
 
+  const tailKey = `${opts.taskId}:${opts.agentId}`;
+
   // ---- Attach: a previous host process left this agent's runner alive. ----
   if (opts.allowAttach && row && row.status === "running" && heartbeatFresh(row.sessionDir)) {
     const proc = new DurableProcess({
@@ -127,7 +148,7 @@ export function createAgentTransport(opts: AgentTransportOptions): LineTransport
       debugLabel: opts.debugLabel,
     });
     proc.onExit(markClosed);
-    return proc;
+    return claimTail(tailKey, proc);
   }
 
   // ---- Spawn a fresh runner, asking any predecessor to stop. --------------
@@ -231,7 +252,7 @@ export function createAgentTransport(opts: AgentTransportOptions): LineTransport
     .run();
 
   proc.onExit(markClosed);
-  return proc;
+  return claimTail(tailKey, proc);
 }
 
 function heartbeatFresh(sessionDir: string): boolean {

package/lib/orchestrator.ts

@@ -688,7 +688,11 @@ class Orchestrator {
         // becomes a zombie: the map still holds the dead session and every
         // later deliver "succeeds" into a closed pipe (found live 2026-07-08
         // after a double SIGKILL). Bounded by the respawn budget, checked in
-        // reconcileSessions.
+        // reconcileSessions. NOTE (durable sessions): an errored TURN does
+        // not mean a dead RUNNER — the dropped session's outbox tail would
+        // poll forever and duplicate every later turn (×5 live 2026-07-20).
+        // createAgentTransport's one-tail-per-agent registry detaches the
+        // stale consumer when the respawn attaches.
         channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
         channel.respawnLastAt.set(agentId, Date.now());
         channel.sessions.delete(agentId);
@@ -1459,7 +1463,12 @@ async function dockerListContainersByLabel(
     { timeoutMs: 60_000 },
   );
   if (res.status === null) return null; // docker state UNKNOWN — don't act on it
-  if (res.status !== 0) return [];
+  // Non-zero is ALSO unknown, not "no containers": after a machine reboot the
+  // host service races the Docker daemon, and `docker ps` fails fast with
+  // "cannot connect". Treating that as [] downgraded every active task to
+  // stopped on boot (live user report 2026-07-21) — the exited-container
+  // restart branch below never got a chance to run.
+  if (res.status !== 0) return null;
   return res.stdout
     .split("\n")
     .map((l) => l.trim())
@@ -1514,7 +1523,40 @@ async function dockerExec(
 }
 
 /** Exported for tests; production entry is the getOrchestrator() boot guard. */
-export async function recoverRunningTasks(): Promise<void> {
+export async function recoverRunningTasks(opts?: {
+  /** Delay between passes while docker is unreachable (tests shrink it). */
+  retryMs?: number;
+  /** Bound on passes; ~15 min at the default cadence covers slow boots. */
+  maxPasses?: number;
+}): Promise<void> {
+  const retryMs = opts?.retryMs ?? 20_000;
+  const maxPasses = opts?.maxPasses ?? 45;
+  // After a machine reboot the host service and the Docker daemon start
+  // concurrently; a pass that finds docker unreachable resolves nothing and
+  // must retry once the daemon is up — the whole point of recovery is to
+  // restart the exited containers that reboot left behind.
+  for (let pass = 1; ; pass++) {
+    const deferred = await recoveryPass();
+    if (deferred === 0) return;
+    if (pass >= maxPasses) {
+      console.warn(
+        `[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred} task(s) unresolved)`,
+      );
+      return;
+    }
+    if (pass === 1) {
+      console.log(
+        `[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred} task(s)`,
+      );
+    }
+    await new Promise((resolve) => setTimeout(resolve, retryMs));
+  }
+}
+
+/** One recovery sweep. Returns how many tasks were deferred on an
+ *  unreachable docker (0 = everything resolved one way or another). */
+async function recoveryPass(): Promise<number> {
+  let deferred = 0;
   try {
     const db = getDb();
     const rows = db
@@ -1522,13 +1564,13 @@ export async function recoverRunningTasks(): Promise<void> {
       .from(schema.hostTasks)
       .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
       .all();
-    if (rows.length === 0) return;
+    if (rows.length === 0) return 0;
     console.log(
       `[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
     );
     for (const task of rows) {
       try {
-        await recoverOneTask(task);
+        if (!(await recoverOneTask(task))) deferred += 1;
       } catch (err) {
         console.error(
           `[orchestrator] recovery: ${task.taskId} failed:`,
@@ -1542,17 +1584,19 @@ export async function recoverRunningTasks(): Promise<void> {
       err instanceof Error ? err.message : err,
     );
   }
+  return deferred;
 }
 
+/** Returns false when docker was unreachable (caller retries the pass). */
 async function recoverOneTask(
   task: typeof schema.hostTasks.$inferSelect,
-): Promise<void> {
+): Promise<boolean> {
   const composeProject = task.composeProject;
   if (!composeProject) {
     // We never wrote a compose project name for this row — must be
     // a row stuck in `queued`/`starting` from before task-up got past
     // step 2. Leave it; the user can recreate.
-    return;
+    return true;
   }
   const containerName = `${composeProject}-app-1`;
   const containers = await dockerListContainersByLabel(
@@ -1560,11 +1604,11 @@ async function recoverOneTask(
   );
   if (containers === null) {
     // Docker unreachable / timed out — actual state unknown. Leave the row
-    // alone; the next recovery pass (or the next ensure) will see the truth.
+    // alone; the retrying caller re-scans once docker answers.
     console.warn(
-      `[orchestrator] recovery: ${task.taskId} skipped — docker did not answer`,
+      `[orchestrator] recovery: ${task.taskId} deferred — docker did not answer`,
     );
-    return;
+    return false;
   }
 
   if (containers.length === 0) {
@@ -1581,7 +1625,7 @@ async function recoverOneTask(
       `[orchestrator] recovery: ${task.taskId} -> ${next} (container gone, ` +
         `worktree ${workspaceOnDisk ? "present" : "gone"})`,
     );
-    return;
+    return true;
   }
 
   const running = containers.some((c) => c.State === "running");
@@ -1603,7 +1647,7 @@ async function recoverOneTask(
     if (task.ownerUserId) {
       void setupTaskGithub(task.taskId, task.ownerUserId);
     }
-    return;
+    return true;
   }
 
   // Container exists but exited. Bring it back up + re-launch the
@@ -1616,7 +1660,7 @@ async function recoverOneTask(
       codeServerPort: null,
       previewPorts: "[]",
     });
-    return;
+    return true;
   }
   // Correctly-owned Codex creds BEFORE uai-init / any agent respawn: the boot
   // reinject sweep only targets containers already running, so a container
@@ -1645,6 +1689,7 @@ async function recoverOneTask(
   console.log(
     `[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
   );
+  return true;
 }
 
 function db_setStatus(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.8.17",
+  "version": "0.8.19",
   "description": "Uai host — runs ephemeral AI coding tasks in Docker on a machine you control.",
   "license": "MIT",
   "author": "Diogo Perillo <diogo.perillo@gmail.com>",