@runuai/host 0.2.4 → 0.2.6

package/lib/docker-exec.ts

@@ -59,6 +59,13 @@ export function dockerCli(
     child.on("close", (code) => {
       finish(code);
     });
+    // EPIPE guard: if the child exits without draining stdin (or our own
+    // timeout SIGKILLs it mid-write), the failed write surfaces as an 'error'
+    // event on the stdin stream — unhandled, that's an uncaughtException that
+    // would take down the whole host service. Today's inputs are tiny (a gh
+    // token) and complete synchronously, but this keeps the helper safe for
+    // any future caller piping real data.
+    child.stdin?.on("error", () => {});
     if (opts.input !== undefined) child.stdin?.write(opts.input);
     child.stdin?.end();
   });

package/lib/github-tokens.ts

@@ -16,7 +16,11 @@ import type { CloudToHost } from "../src/protocol";
 
 const REFRESH_LEAD_MS = 5 * 60 * 1000; // refresh 5 min before expiry
 const EXEC_TIMEOUT_MS = 10_000;
-const EXCHANGE_TIMEOUT_MS = 15_000; // cap the cloud token-exchange round-trip
+// Cap the cloud token-exchange round-trip. Generous on purpose: aborting an
+// exchange that GitHub already completed burns the single-use refresh token
+// (we never see the rotated replacement), so a slow success beats a fast
+// ambiguous abort.
+const EXCHANGE_TIMEOUT_MS = 30_000;
 
 type GhConnectSet = Extract<CloudToHost, { kind: "gh.connect.set" }>;
 
@@ -25,6 +29,9 @@ type GhConnectSet = Extract<CloudToHost, { kind: "gh.connect.set" }>;
 /** Encrypt + persist the user's refresh token (gh.connect.set handler). */
 export function onConnectSet(frame: GhConnectSet): { ok: boolean; error?: string } {
   try {
+    // Fresh grant — a cached access token from the previous grant may be
+    // revoked; drop it so the next mint exchanges against the new token.
+    clearAccessCache(frame.userId);
     const sealed = sealAesGcm(frame.refreshToken);
     const now = Date.now();
     getDb()
@@ -64,6 +71,7 @@ export function onConnectClear(userId: string): void {
 }
 
 export function deleteToken(userId: string): void {
+  clearAccessCache(userId);
   getDb()
     .delete(schema.githubTokens)
     .where(eq(schema.githubTokens.userId, userId))
@@ -115,8 +123,68 @@ interface ExchangeResult {
  * Mint a fresh access token for the user via the cloud exchange. Persists a
  * rotated refresh token if GitHub returned one. Returns null when the user has
  * no token on this host. Throws on exchange failure.
+ *
+ * Deduped PER USER: GitHub refresh tokens are single-use (each exchange
+ * rotates them), and GitHub treats reuse of a consumed token as a compromise
+ * signal that can kill the whole grant. Concurrent task setups for the same
+ * user (several task-ups at once, or every channel re-ensure after a host
+ * restart) used to race the rotation — the loser burned the grant and every
+ * exchange afterwards failed until a manual re-connect. The access token is
+ * per-user anyway, so concurrent callers share one in-flight exchange.
+ */
+const inflightExchanges = new Map<
+  string,
+  Promise<{ accessToken: string; expiresAt: number } | null>
+>();
+
+/**
+ * Per-user ACCESS-token cache. Access tokens live ~8h, but every exchange
+ * ROTATES the single-use refresh token — and a rotation whose response is
+ * lost mid-flight (our timeout firing while GitHub already rotated) burns the
+ * grant, forcing a manual re-connect. So exchange as rarely as possible: all
+ * task setups and refresh timers for a user share one cached access token
+ * until it nears expiry. The refresh timers fire at expiry−5min, inside the
+ * 15min stale window, so exactly one of them performs the real exchange (the
+ * in-flight dedupe serializes any ties) and the rest re-inject from cache.
  */
-export async function requestAccessToken(
+const ACCESS_CACHE_LEAD_MS = 15 * 60 * 1000;
+const accessCache = new Map<
+  string,
+  { accessToken: string; expiresAt: number }
+>();
+
+/** Drop a user's cached access token (token deleted / re-granted). */
+function clearAccessCache(userId: string): void {
+  accessCache.delete(userId);
+}
+
+/** Test hook: reset the access-token cache. */
+export function clearAllAccessCache(): void {
+  accessCache.clear();
+}
+
+export function requestAccessToken(
+  userId: string,
+): Promise<{ accessToken: string; expiresAt: number } | null> {
+  const cached = accessCache.get(userId);
+  if (cached && cached.expiresAt - Date.now() > ACCESS_CACHE_LEAD_MS) {
+    return Promise.resolve(cached);
+  }
+  const existing = inflightExchanges.get(userId);
+  if (existing) return existing;
+  const run = doRequestAccessToken(userId)
+    .then((tok) => {
+      if (tok) accessCache.set(userId, tok);
+      return tok;
+    })
+    .finally(() => {
+      inflightExchanges.delete(userId);
+    });
+  inflightExchanges.set(userId, run);
+  return run;
+}
+
+async function doRequestAccessToken(
   userId: string,
 ): Promise<{ accessToken: string; expiresAt: number } | null> {
   const row = getDb()
@@ -140,7 +208,19 @@ export async function requestAccessToken(
     signal: AbortSignal.timeout(EXCHANGE_TIMEOUT_MS),
   });
   if (!res.ok) {
-    throw new Error(`gh token exchange failed: HTTP ${res.status}`);
+    // Include the cloud's error body: it passes GitHub's OAuth error code
+    // through (e.g. "bad_refresh_token"/"invalid_grant" inside the 502
+    // message), which is the ONLY reliable revocation signal. The bare HTTP
+    // status is not one — the cloud returns 401 solely for a host-token
+    // mismatch and wraps every GitHub-side failure as 502.
+    let detail = "";
+    try {
+      const body = (await res.json()) as { message?: string };
+      if (body && typeof body.message === "string") detail = ` — ${body.message}`;
+    } catch {
+      // non-JSON body — status alone will have to do
+    }
+    throw new Error(`gh token exchange failed: HTTP ${res.status}${detail}`);
   }
   const data = (await res.json()) as ExchangeResult;
   if (data.refreshToken) {
@@ -226,6 +306,19 @@ export function scheduleRefresh(
   timers.set(taskId, timer);
 }
 
+/**
+ * True when the exchange error names a GitHub-side revocation — the refresh
+ * token itself is dead, so retrying is pointless and the user must re-grant.
+ * Deliberately keyed on GitHub's OAuth error strings (passed through the
+ * cloud's 502 body by requestAccessToken), NOT on bare HTTP status: the cloud
+ * returns 401 only for a host-token mismatch and wraps all GitHub failures as
+ * 502, so a status like "401"/"403" alone never means "revoked" — the old
+ * \b40[13]\b match here deleted VALID refresh tokens on host-auth/proxy noise.
+ */
+function isRevokedTokenError(reason: string): boolean {
+  return /revoked|invalid_grant|bad_refresh/i.test(reason);
+}
+
 async function runRefresh(taskId: string, userId: string): Promise<void> {
   try {
     const tok = await requestAccessToken(userId);
@@ -240,7 +333,7 @@ async function runRefresh(taskId: string, userId: string): Promise<void> {
     // A revoked / expired refresh token can't recover — drop it to force a
     // clean re-grant through the OAuth flow. Anything else is likely transient
     // (network / cloud blip) — self-heal with a bounded retry.
-    if (/\b40[13]\b|revoked|invalid_grant|bad_refresh/i.test(reason)) {
+    if (isRevokedTokenError(reason)) {
       deleteToken(userId);
     } else {
       scheduleGithubRetry(taskId, userId, 0);
@@ -276,22 +369,42 @@ function clearGithubRetry(taskId: string): void {
   }
 }
 
+/** Is the task still live on this host? Retries for ended tasks would docker
+ *  exec against a removed container and post notes into a finished chat. */
+function taskIsActive(taskId: string): boolean {
+  const row = getDb()
+    .select({ endedAt: schema.hostTasks.endedAt })
+    .from(schema.hostTasks)
+    .where(eq(schema.hostTasks.taskId, taskId))
+    .get();
+  return row != null && row.endedAt == null;
+}
+
 function scheduleGithubRetry(
   taskId: string,
   userId: string,
   attempt: number,
+  deps: SetupTaskDeps = {},
 ): void {
   clearGithubRetry(taskId);
   if (attempt >= RETRY_BACKOFF_MS.length) {
+    // attempt counts retries fired so far; +1 for the initial try.
     console.warn(
-      `[github] task ${taskId}: gh setup still failing after ${attempt} attempts — giving up until /retry-gh`,
+      `[github] task ${taskId}: gh setup still failing after ${attempt + 1} attempts — giving up until /retry-gh`,
+    );
+    authExpiredHandler?.(
+      taskId,
+      userId,
+      "gh setup kept failing — send /retry-gh once the connection recovers",
     );
     return;
   }
   const delay = RETRY_BACKOFF_MS[attempt] ?? 300_000;
   const timer = setTimeout(() => {
     retryTimers.delete(taskId);
-    void setupTaskGithub(taskId, userId, {}, attempt + 1);
+    const isActive = deps.taskIsActive ?? taskIsActive;
+    if (!isActive(taskId)) return; // task ended while we backed off
+    void setupTaskGithub(taskId, userId, deps, attempt + 1);
   }, delay);
   timer.unref?.();
   retryTimers.set(taskId, timer);
@@ -313,6 +426,8 @@ export interface SetupTaskDeps {
   ) => Promise<{ accessToken: string; expiresAt: number } | null>;
   inject?: (taskId: string, token: string) => void | Promise<void>;
   schedule?: (taskId: string, userId: string, expiresAt: number) => void;
+  deleteToken?: (userId: string) => void;
+  taskIsActive?: (taskId: string) => boolean;
 }
 
 /**
@@ -363,10 +478,19 @@ export async function setupTaskGithub(
   } catch (err) {
     const reason = err instanceof Error ? err.message : String(err);
     console.warn(`[github] task ${taskId}: gh setup failed: ${reason}`);
-    authExpiredHandler?.(taskId, userId, reason);
-    // Self-heal: a transient exchange/inject failure shouldn't strand the task
-    // on a stale token until a manual /retry-gh. Retry with bounded backoff.
-    scheduleGithubRetry(taskId, userId, attempt);
+    // A revoked refresh token can't recover by retrying — drop it so the user
+    // gets a clean re-grant, and tell the chat once.
+    if (isRevokedTokenError(reason)) {
+      (deps.deleteToken ?? deleteToken)(userId);
+      authExpiredHandler?.(taskId, userId, reason);
+      return false;
+    }
+    // Transient: self-heal with bounded backoff. Post the chat note only on
+    // the FIRST failure of a chain — each retry re-enters this catch, and six
+    // "gh auth" notes for one outage is noise. Exhaustion posts its own note
+    // (in scheduleGithubRetry).
+    if (attempt === 0) authExpiredHandler?.(taskId, userId, reason);
+    scheduleGithubRetry(taskId, userId, attempt, deps);
     return false;
   } finally {
     setupInFlight.delete(taskId);

package/lib/orchestrator.ts

@@ -56,8 +56,19 @@ interface Channel {
   /** Per-agent first-turn message — only agents whose `initialPrompt` is
    *  non-empty have an entry. Delivered once at container-ready. */
   firstTurns: Map<string, string>;
-  /** True once agent sessions have been spawned (lazily, on first send). */
-  sessionsStarted: boolean;
+  /** In-flight (or completed) session start. A memoized promise — not a
+   *  boolean flag — so a concurrent deliver() awaits actual readiness instead
+   *  of observing "started" while the sessions map is still empty (the start
+   *  awaits docker work before populating it). Reset to null on failure so the
+   *  next ensure retries. */
+  sessionsReady: Promise<boolean> | null;
+  /** Agents with turn output already streamed for the current turn — i.e. the
+   *  cloud has buffered content for them (see ChannelRouter.turnBuffer). */
+  openTurns: Set<string>;
+  /** Agents whose current turn was interrupted (ESC) — their next
+   *  turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
+   *  half-turn instead of delivering it to @-mentioned peers. */
+  interrupted: Set<string>;
   /** Per-agent respawn counter — bounded so a broken agent can't
    *  loop forever rewriting its config. */
   respawns: Map<string, number>;
@@ -149,7 +160,9 @@ class Orchestrator {
       containerName: `task-${taskId.toLowerCase()}-app-1`,
       preambles,
       firstTurns,
-      sessionsStarted: false,
+      sessionsReady: null,
+      openTurns: new Set(),
+      interrupted: new Set(),
       respawns: new Map(),
     };
     this.channels.set(taskId, channel);
@@ -167,16 +180,31 @@ class Orchestrator {
    *
    * Returns whether sessions are ready.
    */
-  private async ensureSessions(channel: Channel): Promise<boolean> {
-    if (channel.sessionsStarted) return true;
+  private ensureSessions(channel: Channel): Promise<boolean> {
+    // Memoized: every caller awaits the SAME in-flight start, so a concurrent
+    // deliver() can't observe "ready" while the sessions map is still empty
+    // (startSessions awaits docker work before populating it — the old boolean
+    // flag flipped before that await and opened exactly that window).
+    if (!channel.sessionsReady) {
+      channel.sessionsReady = this.startSessions(channel);
+      // A failed/aborted start must not poison the channel — reset so the
+      // next ensure retries (e.g. the task wasn't `running` yet).
+      channel.sessionsReady.then(
+        (ok) => {
+          if (!ok) channel.sessionsReady = null;
+        },
+        () => {
+          channel.sessionsReady = null;
+        },
+      );
+    }
+    return channel.sessionsReady;
+  }
 
+  private async startSessions(channel: Channel): Promise<boolean> {
     const task = getHostTask(channel.taskId);
     if (!task || task.statusMirror !== "running") return false;
 
-    // Flip the flag before the await so a concurrent send can't
-    // double-spawn the sessions.
-    channel.sessionsStarted = true;
-
     // Set the task creator's git author identity in the container (ADR-029).
     // The SSH key itself is installed earlier by task-up.sh (host clone +
     // container), using the creator's per-user key. Best-effort. Awaited but
@@ -267,8 +295,15 @@ class Orchestrator {
     taskId: string,
     agentId: string,
   ): Promise<{ ok: true } | { ok: false; error: string }> {
-    const session = this.channels.get(taskId)?.sessions.get(agentId);
-    if (!session) return { ok: false, error: "no active session" };
+    const channel = this.channels.get(taskId);
+    const session = channel?.sessions.get(agentId);
+    if (!channel || !session) return { ok: false, error: "no active session" };
+    // An interrupted turn is a half-turn — its eventual turn_complete must not
+    // hand the buffered fragment to @-mentioned peers. Flag it so the boundary
+    // goes out `aborted` and the cloud discards the buffer. Only when the turn
+    // actually streamed output (openTurns): an idle-agent ESC must not mark
+    // the NEXT legitimate turn as aborted.
+    if (channel.openTurns.has(agentId)) channel.interrupted.add(agentId);
     void session.interrupt();
     this.emitSystemNote(taskId, `Stopped @${agentId}.`);
     return { ok: true };
@@ -300,6 +335,7 @@ class Orchestrator {
   ): Promise<void> {
     switch (event.type) {
       case "message_delta": {
+        channel.openTurns.add(agentId);
         this.emitHost({
           kind: "agent.message_delta",
           taskId: channel.taskId,
@@ -309,6 +345,7 @@ class Orchestrator {
         break;
       }
       case "message_complete": {
+        channel.openTurns.add(agentId);
         this.emitHost({
           kind: "agent.message_complete",
           taskId: channel.taskId,
@@ -354,6 +391,10 @@ class Orchestrator {
         break;
       }
       case "error": {
+        // The turn died with the session — drop its turn-state flags so a
+        // respawned session starts clean.
+        channel.openTurns.delete(agentId);
+        channel.interrupted.delete(agentId);
         // Claude under load (especially Docker Desktop macOS) occasionally
         // unlinks ~/.claude.json mid-write during atomic config rewrites,
         // and a concurrent claude spawn lands during the gap and exits 0
@@ -381,10 +422,15 @@ class Orchestrator {
       case "turn_complete": {
         // Turn boundary — the cloud flushes any @-mentions buffered across this
         // turn's messages and wakes the mentioned peers with the full turn.
+        // An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
+        // the cloud discards the buffer instead of handing it to peers.
+        const aborted = channel.interrupted.delete(agentId);
+        channel.openTurns.delete(agentId);
         this.emitHost({
           kind: "agent.turn_complete",
           taskId: channel.taskId,
           agentId,
+          aborted,
         });
         break;
       }
@@ -817,15 +863,21 @@ interface DockerPs {
   State: string; // "running" | "exited" | "created" | "paused"
 }
 
-async function dockerListContainersByLabel(label: string): Promise<DockerPs[]> {
-  const res = await dockerCli([
-    "ps",
-    "--all",
-    "--filter",
-    `label=${label}`,
-    "--format",
-    "{{json .}}",
-  ]);
+/**
+ * Returns the task's containers, [] when docker reports none, or NULL when
+ * docker itself was unreachable/timed out (status null = spawn failure or our
+ * SIGKILL). The distinction matters: recovery treats [] as "container gone"
+ * and DOWNGRADES the task to stopped/error — doing that because a `docker ps`
+ * timed out under load would destroy a perfectly recoverable task.
+ */
+async function dockerListContainersByLabel(
+  label: string,
+): Promise<DockerPs[] | null> {
+  const res = await dockerCli(
+    ["ps", "--all", "--filter", `label=${label}`, "--format", "{{json .}}"],
+    { timeoutMs: 60_000 },
+  );
+  if (res.status === null) return null; // docker state UNKNOWN — don't act on it
   if (res.status !== 0) return [];
   return res.stdout
     .split("\n")
@@ -842,7 +894,9 @@ async function dockerListContainersByLabel(label: string): Promise<DockerPs[]> {
 }
 
 async function dockerStart(containerName: string): Promise<boolean> {
-  const res = await dockerCli(["start", containerName]);
+  // Starting a big dev container can legitimately take a while — give it far
+  // more than dockerCli's 30s default before declaring the task stopped.
+  const res = await dockerCli(["start", containerName], { timeoutMs: 120_000 });
   if (res.status !== 0) {
     console.error(
       `[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
@@ -866,8 +920,15 @@ async function dockerPort(
   return Number.isFinite(port) ? port : null;
 }
 
-async function dockerExec(containerName: string, cmd: string[]): Promise<boolean> {
-  const res = await dockerCli(["exec", containerName, ...cmd]);
+async function dockerExec(
+  containerName: string,
+  cmd: string[],
+  timeoutMs?: number,
+): Promise<boolean> {
+  const res = await dockerCli(
+    ["exec", containerName, ...cmd],
+    timeoutMs === undefined ? {} : { timeoutMs },
+  );
   return res.status === 0;
 }
 
@@ -915,6 +976,14 @@ async function recoverOneTask(
   const containers = await dockerListContainersByLabel(
     `com.docker.compose.project=${composeProject}`,
   );
+  if (containers === null) {
+    // Docker unreachable / timed out — actual state unknown. Leave the row
+    // alone; the next recovery pass (or the next ensure) will see the truth.
+    console.warn(
+      `[orchestrator] recovery: ${task.taskId} skipped — docker did not answer`,
+    );
+    return;
+  }
 
   if (containers.length === 0) {
     // Container gone. Worktree state determines whether resume is
@@ -945,6 +1014,13 @@ async function recoverOneTask(
       });
       console.log(`[orchestrator] recovery: ${task.taskId} ports refreshed`);
     }
+    // Re-establish gh proactively (ADR-027): the restart killed the in-memory
+    // refresh timers, so an idle task's access token would silently expire
+    // ~8h later. Cheap with the per-user access-token cache — one exchange
+    // per user, every task re-injects from it. Best-effort.
+    if (task.ownerUserId) {
+      void setupTaskGithub(task.taskId, task.ownerUserId);
+    }
     return;
   }
 
@@ -960,7 +1036,11 @@ async function recoverOneTask(
     });
     return;
   }
-  if (!(await dockerExec(containerName, ["/usr/local/bin/uai-init"]))) {
+  // uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
+  // repo. dockerCli's 30s default would SIGKILL it mid-install.
+  if (
+    !(await dockerExec(containerName, ["/usr/local/bin/uai-init"], 10 * 60_000))
+  ) {
     console.warn(
       `[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
     );
@@ -969,6 +1049,11 @@ async function recoverOneTask(
   db_setRuntime(task.taskId, {
     codeServerPort: port,
   });
+  // Same gh re-establish as the running branch — the restarted container's gh
+  // config is whatever it held when it exited.
+  if (task.ownerUserId) {
+    void setupTaskGithub(task.taskId, task.ownerUserId);
+  }
   console.log(
     `[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
   );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "Uai host — runs ephemeral AI coding tasks in Docker on a machine you control.",
   "license": "MIT",
   "author": "Diogo Perillo <diogo.perillo@gmail.com>",

package/src/protocol.ts

@@ -402,11 +402,14 @@ export type HostEvent =
   // The agent finished a turn (it may have emitted several message_complete
   // items within it). The cloud waits for this before waking @-mentioned peers,
   // so a peer is handed the agent's COMPLETE turn rather than a mid-turn
-  // fragment that happened to contain the mention.
+  // fragment that happened to contain the mention. `aborted` marks a turn that
+  // was interrupted (ESC): the cloud discards the buffered half-turn instead
+  // of delivering it.
   | {
       kind: "agent.turn_complete";
       taskId: string;
       agentId: string;
+      aborted?: boolean;
     }
   | {
       kind: "agent.tool_call";