@runuai/host 0.2.2 → 0.2.4

package/lib/docker-exec.ts

@@ -0,0 +1,65 @@
+/**
+ * Async docker CLI runner.
+ *
+ * The host's hot paths — boot recovery (recoverRunningTasks) and the
+ * per-reconnect channel ensure (git identity + gh token injection) — used
+ * `spawnSync`, which blocks the single Node event loop for the entire docker
+ * call. With several tasks (and large containers), those blocking bursts starve
+ * the bridge heartbeat timer and the local UI server: the cloud stops getting
+ * pings, drops the host, the host reconnects, the cloud re-`ensure`s the
+ * channels → more blocking docker → the heartbeat dies again. A reconnect loop.
+ *
+ * Running docker through async `spawn` keeps the loop responsive while the
+ * command runs. Never rejects — it resolves the same `{status, stdout, stderr}`
+ * shape callers already branch on (mirrors spawnSync's result fields), with
+ * `status: null` on spawn error or timeout.
+ */
+
+import { spawn } from "node:child_process";
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+export interface DockerResult {
+  status: number | null;
+  stdout: string;
+  stderr: string;
+}
+
+export function dockerCli(
+  args: string[],
+  opts: { input?: string; timeoutMs?: number } = {},
+): Promise<DockerResult> {
+  return new Promise<DockerResult>((resolve) => {
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    const child = spawn("docker", args, { stdio: ["pipe", "pipe", "pipe"] });
+    const timer = setTimeout(() => {
+      child.kill("SIGKILL");
+    }, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    timer.unref?.();
+    const finish = (status: number | null) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve({ status, stdout, stderr });
+    };
+    child.stdout?.setEncoding("utf8");
+    child.stderr?.setEncoding("utf8");
+    child.stdout?.on("data", (c: string) => {
+      stdout += c;
+    });
+    child.stderr?.on("data", (c: string) => {
+      stderr += c;
+    });
+    child.on("error", (err: Error) => {
+      stderr += err.message;
+      finish(null);
+    });
+    child.on("close", (code) => {
+      finish(code);
+    });
+    if (opts.input !== undefined) child.stdin?.write(opts.input);
+    child.stdin?.end();
+  });
+}

package/lib/git-identity.ts

@@ -5,30 +5,27 @@
  * task-up. Name falls back to the email when no display name is known.
  */
 
-import { spawnSync } from "node:child_process";
+import { dockerCli } from "./docker-exec";
 
 const EXEC_TIMEOUT_MS = 10_000;
 
-/** Injectable docker-exec seam (mocked in tests). */
-export type DockerExec = (args: string[]) => {
-  status: number | null;
-  stderr: string;
-};
+/** Injectable docker-exec seam (mocked in tests). Async so the docker call
+ *  doesn't block the host event loop (this runs on every channel ensure). */
+export type DockerExec = (
+  args: string[],
+) => Promise<{ status: number | null; stderr: string }>;
 
-const defaultExec: DockerExec = (args) => {
-  const res = spawnSync("docker", args, {
-    encoding: "utf8",
-    timeout: EXEC_TIMEOUT_MS,
-  });
-  return { status: res.status, stderr: res.stderr ?? "" };
+const defaultExec: DockerExec = async (args) => {
+  const res = await dockerCli(args, { timeoutMs: EXEC_TIMEOUT_MS });
+  return { status: res.status, stderr: res.stderr };
 };
 
-export function setupTaskGitIdentity(
+export async function setupTaskGitIdentity(
   taskId: string,
   name: string | null,
   email: string | null,
   exec: DockerExec = defaultExec,
-): boolean {
+): Promise<boolean> {
   if (!email) {
     console.log(`[git] task ${taskId}: no owner email — leaving git identity`);
     return false;
@@ -42,7 +39,7 @@ export function setupTaskGitIdentity(
     ["user.email", email],
   ];
   for (const [key, value] of entries) {
-    const res = exec([
+    const res = await exec([
       "exec",
       "-u",
       "node",

package/lib/github-tokens.ts

@@ -7,12 +7,11 @@
  * exchange is a host→cloud HTTP POST to /api/github/oauth/exchange.
  */
 
-import { spawnSync } from "node:child_process";
-
 import { and, eq, isNull } from "drizzle-orm";
 
 import { getDb, schema } from "./db";
 import { sealAesGcm, openAesGcm } from "./secrets";
+import { dockerCli } from "./docker-exec";
 import type { CloudToHost } from "../src/protocol";
 
 const REFRESH_LEAD_MS = 5 * 60 * 1000; // refresh 5 min before expiry
@@ -163,27 +162,24 @@ export async function requestAccessToken(
 
 // --- container injection ----------------------------------------------------
 
-/** Injectable docker-exec seam (mocked in tests). */
+/** Injectable docker-exec seam (mocked in tests). Async so the docker call
+ *  doesn't block the host event loop (runs on every channel ensure + refresh). */
 export type DockerExec = (
   args: string[],
   input: string,
-) => { status: number | null; stderr: string };
+) => Promise<{ status: number | null; stderr: string }>;
 
-const defaultExec: DockerExec = (args, input) => {
-  const res = spawnSync("docker", args, {
-    input,
-    encoding: "utf8",
-    timeout: EXEC_TIMEOUT_MS,
-  });
-  return { status: res.status, stderr: res.stderr ?? "" };
+const defaultExec: DockerExec = async (args, input) => {
+  const res = await dockerCli(args, { input, timeoutMs: EXEC_TIMEOUT_MS });
+  return { status: res.status, stderr: res.stderr };
 };
 
 /** Write the access token into the container's gh config via `gh auth login`. */
-export function injectIntoContainer(
+export async function injectIntoContainer(
   taskId: string,
   accessToken: string,
   exec: DockerExec = defaultExec,
-): void {
+): Promise<void> {
   const container = `task-${taskId}-app-1`;
   const args = [
     "exec",
@@ -198,7 +194,7 @@ export function injectIntoContainer(
     "--hostname",
     "github.com",
   ];
-  const res = exec(args, `${accessToken}\n`);
+  const res = await exec(args, `${accessToken}\n`);
   if (res.status !== 0) {
     throw new Error(`gh auth login failed in ${container}: ${res.stderr.trim()}`);
   }
@@ -237,20 +233,24 @@ async function runRefresh(taskId: string, userId: string): Promise<void> {
       authExpiredHandler?.(taskId, userId, "no GitHub token on host");
       return;
     }
-    injectIntoContainer(taskId, tok.accessToken);
+    await injectIntoContainer(taskId, tok.accessToken);
     scheduleRefresh(taskId, userId, tok.expiresAt);
   } catch (err) {
     const reason = err instanceof Error ? err.message : String(err);
     // A revoked / expired refresh token can't recover — drop it to force a
-    // clean re-grant through the OAuth flow.
+    // clean re-grant through the OAuth flow. Anything else is likely transient
+    // (network / cloud blip) — self-heal with a bounded retry.
     if (/\b40[13]\b|revoked|invalid_grant|bad_refresh/i.test(reason)) {
       deleteToken(userId);
+    } else {
+      scheduleGithubRetry(taskId, userId, 0);
     }
     authExpiredHandler?.(taskId, userId, reason);
   }
 }
 
 export function clearRefresh(taskId: string): void {
+  clearGithubRetry(taskId);
   const t = timers.get(taskId);
   if (t) {
     clearTimeout(t);
@@ -258,6 +258,50 @@ export function clearRefresh(taskId: string): void {
   }
 }
 
+// --- self-heal: retry a failed gh setup with bounded backoff ----------------
+// A gh setup/refresh can fail transiently — e.g. the cloud token-exchange fetch
+// times out while the host is briefly unreachable (as happened when the bridge
+// was flapping). Without a retry the task is stranded on a stale/invalid token
+// until someone runs /retry-gh, because ensureSessions won't re-inject once a
+// channel's sessions are started. Retry with bounded backoff; cleared on
+// success (clearGithubRetry) and on task teardown (via clearRefresh).
+const RETRY_BACKOFF_MS = [5_000, 20_000, 60_000, 180_000, 300_000];
+const retryTimers = new Map<string, ReturnType<typeof setTimeout>>();
+
+function clearGithubRetry(taskId: string): void {
+  const t = retryTimers.get(taskId);
+  if (t) {
+    clearTimeout(t);
+    retryTimers.delete(taskId);
+  }
+}
+
+function scheduleGithubRetry(
+  taskId: string,
+  userId: string,
+  attempt: number,
+): void {
+  clearGithubRetry(taskId);
+  if (attempt >= RETRY_BACKOFF_MS.length) {
+    console.warn(
+      `[github] task ${taskId}: gh setup still failing after ${attempt} attempts — giving up until /retry-gh`,
+    );
+    return;
+  }
+  const delay = RETRY_BACKOFF_MS[attempt] ?? 300_000;
+  const timer = setTimeout(() => {
+    retryTimers.delete(taskId);
+    void setupTaskGithub(taskId, userId, {}, attempt + 1);
+  }, delay);
+  timer.unref?.();
+  retryTimers.set(taskId, timer);
+}
+
+/** Test/diagnostic: is a gh-setup retry currently armed for this task? */
+export function hasPendingGithubRetry(taskId: string): boolean {
+  return retryTimers.has(taskId);
+}
+
 /**
  * Injectable seams for {@link setupTaskGithub} (mocked in tests so the decision
  * logic needs neither the host DB nor docker). All default to the real impls.
@@ -267,7 +311,7 @@ export interface SetupTaskDeps {
   requestAccessToken?: (
     userId: string,
   ) => Promise<{ accessToken: string; expiresAt: number } | null>;
-  inject?: (taskId: string, token: string) => void;
+  inject?: (taskId: string, token: string) => void | Promise<void>;
   schedule?: (taskId: string, userId: string, expiresAt: number) => void;
 }
 
@@ -287,6 +331,7 @@ export async function setupTaskGithub(
   taskId: string,
   userId: string,
   deps: SetupTaskDeps = {},
+  attempt = 0,
 ): Promise<boolean> {
   const _hasToken = deps.hasToken ?? hasToken;
   const _request = deps.requestAccessToken ?? requestAccessToken;
@@ -299,15 +344,17 @@ export async function setupTaskGithub(
     if (_hasToken(userId)) {
       const tok = await _request(userId);
       if (tok) {
-        _inject(taskId, tok.accessToken);
+        await _inject(taskId, tok.accessToken);
         _schedule(taskId, userId, tok.expiresAt);
+        clearGithubRetry(taskId);
         console.log(`[github] task ${taskId}: injected user access token`);
         return true;
       }
     }
     const pat = process.env.UAI_GH_PAT_FALLBACK;
     if (pat) {
-      _inject(taskId, pat);
+      await _inject(taskId, pat);
+      clearGithubRetry(taskId);
       console.log(`[github] task ${taskId}: using PAT fallback (no refresh)`);
       return true;
     }
@@ -317,14 +364,19 @@ export async function setupTaskGithub(
     const reason = err instanceof Error ? err.message : String(err);
     console.warn(`[github] task ${taskId}: gh setup failed: ${reason}`);
     authExpiredHandler?.(taskId, userId, reason);
+    // Self-heal: a transient exchange/inject failure shouldn't strand the task
+    // on a stale token until a manual /retry-gh. Retry with bounded backoff.
+    scheduleGithubRetry(taskId, userId, attempt);
     return false;
   } finally {
     setupInFlight.delete(taskId);
   }
 }
 
-/** Test/maintenance hook: drop all scheduled refresh timers. */
+/** Test/maintenance hook: drop all scheduled refresh + retry timers. */
 export function clearAllRefreshTimers(): void {
   for (const t of timers.values()) clearTimeout(t);
   timers.clear();
+  for (const t of retryTimers.values()) clearTimeout(t);
+  retryTimers.clear();
 }

package/lib/orchestrator.ts

@@ -36,6 +36,7 @@ import { ACTIVE_STATUSES } from "./task-status";
 import { getHostTask, upsertHostTask } from "./runtime-state";
 import { setupTaskGithub } from "./github-tokens";
 import { setupTaskGitIdentity } from "./git-identity";
+import { dockerCli } from "./docker-exec";
 import type { ChannelEnsureInput, HostEvent } from "../src/protocol";
 
 export type HostEventSubscriber = (event: HostEvent) => void;
@@ -178,8 +179,10 @@ class Orchestrator {
 
     // Set the task creator's git author identity in the container (ADR-029).
     // The SSH key itself is installed earlier by task-up.sh (host clone +
-    // container), using the creator's per-user key. Best-effort.
-    setupTaskGitIdentity(channel.taskId, task.ownerName, task.ownerEmail);
+    // container), using the creator's per-user key. Best-effort. Awaited but
+    // async (docker exec via dockerCli) so it doesn't block the event loop —
+    // this runs on every channel ensure, including each post-restart reconnect.
+    await setupTaskGitIdentity(channel.taskId, task.ownerName, task.ownerEmail);
 
     // GitHub auth for `gh` (ADR-027): mint + inject the access token and
     // (re)start its refresh schedule. Done here — not only at task-up — so it
@@ -814,12 +817,15 @@ interface DockerPs {
   State: string; // "running" | "exited" | "created" | "paused"
 }
 
-function dockerListContainersByLabel(label: string): DockerPs[] {
-  const res = spawnSync(
-    "docker",
-    ["ps", "--all", "--filter", `label=${label}`, "--format", "{{json .}}"],
-    { encoding: "utf8" },
-  );
+async function dockerListContainersByLabel(label: string): Promise<DockerPs[]> {
+  const res = await dockerCli([
+    "ps",
+    "--all",
+    "--filter",
+    `label=${label}`,
+    "--format",
+    "{{json .}}",
+  ]);
   if (res.status !== 0) return [];
   return res.stdout
     .split("\n")
@@ -835,10 +841,8 @@ function dockerListContainersByLabel(label: string): DockerPs[] {
     .filter((x): x is DockerPs => x !== null);
 }
 
-function dockerStart(containerName: string): boolean {
-  const res = spawnSync("docker", ["start", containerName], {
-    encoding: "utf8",
-  });
+async function dockerStart(containerName: string): Promise<boolean> {
+  const res = await dockerCli(["start", containerName]);
   if (res.status !== 0) {
     console.error(
       `[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
@@ -848,15 +852,11 @@ function dockerStart(containerName: string): boolean {
   return true;
 }
 
-function dockerPort(
+async function dockerPort(
   containerName: string,
   containerPort: number,
-): number | null {
-  const res = spawnSync(
-    "docker",
-    ["port", containerName, String(containerPort)],
-    { encoding: "utf8" },
-  );
+): Promise<number | null> {
+  const res = await dockerCli(["port", containerName, String(containerPort)]);
   if (res.status !== 0) return null;
   // Output: "127.0.0.1:32785\n"
   const firstLine = res.stdout.split("\n")[0]?.trim() ?? "";
@@ -866,10 +866,8 @@ function dockerPort(
   return Number.isFinite(port) ? port : null;
 }
 
-function dockerExec(containerName: string, cmd: string[]): boolean {
-  const res = spawnSync("docker", ["exec", containerName, ...cmd], {
-    encoding: "utf8",
-  });
+async function dockerExec(containerName: string, cmd: string[]): Promise<boolean> {
+  const res = await dockerCli(["exec", containerName, ...cmd]);
   return res.status === 0;
 }
 
@@ -914,7 +912,7 @@ async function recoverOneTask(
     return;
   }
   const containerName = `${composeProject}-app-1`;
-  const containers = dockerListContainersByLabel(
+  const containers = await dockerListContainersByLabel(
     `com.docker.compose.project=${composeProject}`,
   );
 
@@ -940,7 +938,7 @@ async function recoverOneTask(
     // Container is up. Re-discover the host port in case Docker
     // remapped it across restarts (it usually does for ephemeral
     // bindings).
-    const port = dockerPort(containerName, 8080);
+    const port = await dockerPort(containerName, 8080);
     if (port && port !== task.codeServerPort) {
       db_setRuntime(task.taskId, {
         codeServerPort: port,
@@ -955,19 +953,19 @@ async function recoverOneTask(
   console.log(
     `[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
   );
-  if (!dockerStart(containerName)) {
+  if (!(await dockerStart(containerName))) {
     db_setStatus(task.taskId, "stopped", {
       codeServerPort: null,
       previewPorts: "[]",
     });
     return;
   }
-  if (!dockerExec(containerName, ["/usr/local/bin/uai-init"])) {
+  if (!(await dockerExec(containerName, ["/usr/local/bin/uai-init"]))) {
     console.warn(
       `[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
     );
   }
-  const port = dockerPort(containerName, 8080) ?? null;
+  const port = (await dockerPort(containerName, 8080)) ?? null;
   db_setRuntime(task.taskId, {
     codeServerPort: port,
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "Uai host — runs ephemeral AI coding tasks in Docker on a machine you control.",
   "license": "MIT",
   "author": "Diogo Perillo <diogo.perillo@gmail.com>",

package/src/ui/server.ts

@@ -8,7 +8,6 @@
  * so the API can never emit an off-contract shape.
  */
 
-import { spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import {
@@ -27,6 +26,7 @@ import type { ZodType } from "zod";
 import { schema, type Db } from "../../lib/db";
 import { parsePreviewPortRuntimes } from "../../lib/preview-ports";
 import { getCloudState } from "../../lib/cloud-state";
+import { dockerCli } from "../../lib/docker-exec";
 import {
   CloudResponse,
   EventsResponse,
@@ -305,16 +305,14 @@ export async function dockerMemoryBytes(
   composeProject: string,
 ): Promise<number | null> {
   try {
-    const res = spawnSync(
-      "docker",
-      [
-        "stats",
-        "--no-stream",
-        "--format",
-        "{{.MemUsage}}",
-        `${composeProject}-app-1`,
-      ],
-      { encoding: "utf8", timeout: 3_000 },
+    // Async (not spawnSync): this runs per task on every status request, on the
+    // SAME event loop as the cloud-bridge heartbeat. `docker stats --no-stream`
+    // samples for ~1-2s; doing it synchronously for N tasks blocked the loop
+    // long enough that the bridge heartbeat timed out and the cloud dropped the
+    // host into a reconnect loop.
+    const res = await dockerCli(
+      ["stats", "--no-stream", "--format", "{{.MemUsage}}", `${composeProject}-app-1`],
+      { timeoutMs: 3_000 },
     );
     if (res.status !== 0 || !res.stdout) return null;
     const used = res.stdout.trim().split("/")[0]?.trim();