@runuai/host 0.8.41 → 0.8.43

package/lib/preview-sidecar.ts

@@ -14,6 +14,8 @@
  * connect fails, which invalidates it (mirrors the container-IP cache). Declared
  * ports keep their published-port path; only ad-hoc ports use a sidecar.
  */
+import net from "node:net";
+
 import { dockerCli } from "./docker-exec";
 
 /** Port the proxy listens on inside the sidecar (published to a random host port). */
@@ -45,6 +47,86 @@ function proxyScript(appContainer: string, containerPort: number): string {
   );
 }
 
+/** How long to wait for a new sidecar's proxy to start accepting. */
+const READY_TIMEOUT_MS = 10_000;
+const READY_POLL_MS = 100;
+/** A connection that survives this long AFTER connecting has a live upstream. */
+const PROBE_SETTLE_MS = 150;
+/** A handshake that hasn't completed by now isn't going to. */
+const PROBE_CONNECT_MS = 1_000;
+
+/** The bit of `net.Socket` the probe uses — so a test can supply its own. */
+interface SocketLike {
+  on: (event: string, fn: () => void) => unknown;
+  destroy: () => void;
+}
+
+/**
+ * Whether the whole chain behind `hostPort` is live: the proxy is listening AND
+ * its upstream connect succeeded.
+ *
+ * A plain "did the TCP connect succeed" check proves nothing here, which is the
+ * trap this exists for. Docker publishes the port when the CONTAINER starts,
+ * not when the process inside binds — so `docker-proxy` accepts the connection
+ * either way and then closes it, and an HTTP client reports that as
+ * `socket hang up`. The sidecar's proxy behaves identically when it is up but
+ * its own upstream connect fails (`u.on('error',()=>c.destroy())`).
+ *
+ * So the signal is the connection SURVIVING briefly, not opening. Both failure
+ * modes close it immediately; a healthy pipe just sits there waiting for a
+ * request, which is the one thing neither of them does.
+ */
+function probeSidecar(
+  hostPort: number,
+  /** Seam for tests: a socket that never connects can't be produced locally. */
+  connect: (port: number) => SocketLike = (port) =>
+    net.connect(port, "127.0.0.1"),
+): Promise<boolean> {
+  return new Promise((resolve) => {
+    const socket = connect(hostPort);
+    let settled = false;
+    let settleTimer: ReturnType<typeof setTimeout> | undefined;
+    const done = (ok: boolean): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(settleTimer);
+      clearTimeout(connectTimer);
+      socket.destroy();
+      resolve(ok);
+    };
+    // The settle window starts at `connect`, NOT at the call. Starting it here
+    // would count a connection still stuck in the handshake as healthy — the
+    // one state that is neither of the two we can distinguish, and the one a
+    // wedged host is most likely to be in.
+    //
+    // The connect deadline is disarmed at the same moment, or the two overlap:
+    // a connection established at 900ms would be failed at 1000ms, 50ms into a
+    // 150ms settle window it was on course to pass. The deadlines are
+    // sequential — first "did it connect", then "did it stay" — so only one may
+    // be running.
+    socket.on("connect", () => {
+      clearTimeout(connectTimer);
+      settleTimer = setTimeout(() => done(true), PROBE_SETTLE_MS);
+    });
+    const connectTimer = setTimeout(() => done(false), PROBE_CONNECT_MS);
+    socket.on("error", () => done(false));
+    socket.on("close", () => done(false));
+  });
+}
+
+/** Test seam: the probe with an injectable socket. Not used in production. */
+export const probeForTest = probeSidecar;
+
+/** Poll `probeSidecar` until the chain answers, or give up. */
+async function waitUntilReady(hostPort: number): Promise<boolean> {
+  const deadline = Date.now() + READY_TIMEOUT_MS;
+  for (;;) {
+    if (await probeSidecar(hostPort)) return true;
+    if (Date.now() >= deadline) return false;
+    await new Promise((r) => setTimeout(r, READY_POLL_MS));
+  }
+}
+
 async function imageOf(appContainer: string): Promise<string | null> {
   const r = await dockerCli(
     ["inspect", "-f", "{{.Config.Image}}", appContainer],
@@ -74,8 +156,18 @@ async function createSidecar(args: {
   // Reuse a still-running sidecar from a previous host process (--rm keeps it up
   // across a host restart; the in-memory cache was cleared). `docker port` is an
   // exact-name lookup, so no name-filter regex pitfalls.
+  //
+  // PROBED, not just found. A sidecar outlives the app container it proxies to,
+  // so one left over from a previous container generation is still running and
+  // still publishing its port while its route is dead. Reused blind, that
+  // sidecar serves `socket hang up` forever: the recovery path invalidates the
+  // CACHE on a failed connect, then lands right back here and returns the same
+  // corpse. Nothing could break the cycle short of removing it by hand.
   const existing = await publishedPort(name);
-  if (existing !== null) return existing;
+  if (existing !== null) {
+    if (await probeSidecar(existing)) return existing;
+    await dockerCli(["rm", "-f", name], { timeoutMs: 10_000 });
+  }
 
   const image = await imageOf(appContainer);
   if (!image) return null;
@@ -101,6 +193,20 @@ async function createSidecar(args: {
     await dockerCli(["rm", "-f", name], { timeoutMs: 10_000 });
     return null;
   }
+
+  // Do not hand back a port that isn't answering yet. `docker run -d` returns
+  // when the CONTAINER starts and the mapping exists from that instant, but the
+  // node proxy inside has not called `listen` — so every request arriving in
+  // that window is accepted by docker-proxy and dropped.
+  //
+  // A browser makes that window very expensive. It fetches the document, then
+  // fires every sub-resource at once: one page load put ~20 requests into a
+  // ~3s gap and lost all of them, which renders as an unstyled page with broken
+  // images rather than as anything resembling "not ready yet".
+  if (!(await waitUntilReady(hostPort))) {
+    await dockerCli(["rm", "-f", name], { timeoutMs: 10_000 });
+    return null;
+  }
   return hostPort;
 }
 
@@ -133,8 +239,22 @@ export async function ensurePreviewSidecar(args: {
   return pending;
 }
 
-/** Drop any cached sidecar entry on `hostPort` (a tunnel connect failed). No-op
- *  for published-declared / container-IP targets, which aren't in this cache. */
+/**
+ * Drop the cached port for a sidecar whose route just failed. No-op for
+ * published-declared / container-IP targets, which aren't in this cache.
+ *
+ * CACHE-ONLY, deliberately. This fires from the host's `upstream.on("error")`,
+ * which cannot tell a dead sidecar from a client that hit stop mid-download or
+ * a request that tripped the tunnel's ack deadline — both destroy the socket
+ * with ECONNRESET. Removing the container here would let one abandoned asset
+ * kill a sidecar that is serving twenty other requests fine, which is a worse
+ * failure than the one being recovered from.
+ *
+ * Forgetting the port is enough now: the next `createSidecar` finds the
+ * container by name and PROBES it, so a genuinely dead one is removed there,
+ * where the evidence is direct rather than inferred. That is the escape hatch
+ * this used to lack — invalidation alone kept landing back on the same corpse.
+ */
 export function invalidatePreviewSidecar(hostPort: number): void {
   for (const [key, port] of cache) {
     if (port === hostPort) cache.delete(key);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.8.41",
+  "version": "0.8.43",
   "description": "Uai host — runs ephemeral AI coding tasks in Docker on a machine you control.",
   "license": "MIT",
   "author": "Diogo Perillo <diogo.perillo@gmail.com>",

package/scripts/agent/task-down.sh

@@ -59,6 +59,22 @@ if [ -n "$compose_project" ]; then
     # Fallback: tear down by project name only.
     docker compose -p "$compose_project" down -v --rmi local --remove-orphans >/dev/null 2>&1 || true
   fi
+  # `compose down` is intentionally best-effort because an already-absent
+  # stack can make a config-less fallback fail. Destruction itself is not
+  # best-effort: before deleting the worktree or reporting success, prove that
+  # no container remains under the stable compose project name.
+  remaining_containers=""
+  if ! remaining_containers=$(docker ps -aq \
+    --filter "label=com.docker.compose.project=$compose_project"); then
+    emit_err "COMPOSE_DOWN_FAILED" \
+      "could not verify teardown for compose project $compose_project" \
+      "docker ps after compose down"
+  fi
+  if [ -n "$remaining_containers" ]; then
+    emit_err "COMPOSE_DOWN_FAILED" \
+      "compose project $compose_project still has containers after teardown" \
+      "docker ps after compose down"
+  fi
 fi
 
 # -----------------------------------------------------------------------------

package/src/index.ts

@@ -19,7 +19,11 @@ import {
 } from "../lib/runtime-state";
 import { clearTaskGatewayAcl } from "../lib/mcp-gateway";
 import { parsePreviewPortRuntimes } from "../lib/preview-ports";
-import { ensurePreviewSidecar } from "../lib/preview-sidecar";
+import { dockerCli } from "../lib/docker-exec";
+import {
+  ensurePreviewSidecar,
+  stopPreviewSidecars,
+} from "../lib/preview-sidecar";
 import {
   HostErrorCode,
   type CommandContext,
@@ -106,75 +110,180 @@ export const hostEvents: HostEventStream = {
 export const hostCommands: HostCommands = {
   async taskUp(ctx, input) {
     logCommand(ctx, "taskUp", input.task.id);
-    recordTaskStarting(input.task.id);
-    recordTaskOwner(
-      input.task.id,
-      input.task.ownerUserId,
-      input.ownerEmail ?? null,
-      input.ownerName ?? null,
-      input.projects.map((p) => p.slug),
-    );
-    // ADR-048: persist the cloud-issued per-task cli secret (host-private) so the
-    // `uai` CLI's per-agent tokens can be minted now + after a host restart.
-    if (input.task.cliSecret) {
-      storeTaskCliSecret(input.task.id, input.task.cliSecret);
-    }
-    recordHostEvent(input.task.id, "task.created");
-    const result = await wrapAgent(ctx, "taskUp", async () => {
-      // Self-heal the standard base image before task-up needs it. The
-      // boot-time build is best-effort + one-shot: a host that started before
-      // Docker was ready (common on Docker Desktop) never built
-      // uai-standard:dev, so every task-up died on a doomed registry pull
-      // (live 2026-07-22). This is idempotent — a fast inspect when present,
-      // a real build only when missing/stale (Docker is definitely up now, a
-      // task just arrived). On a build FAILURE, surface WHY to the cloud (not
-      // just the host log — the operator can't read a remote user's machine).
-      const img = await ensureStandardImage();
-      if (!img.ok) {
-        throw new AgentError(
-          "STANDARD_IMAGE_UNAVAILABLE",
-          img.error
-            ? `could not prepare the host base image uai-standard:dev — the build failed. ${toolStderrDetail(img.error)}`
-            : "could not prepare the host base image uai-standard:dev — see the host log.",
-          {},
+    const orchestrator = getOrchestrator();
+    return orchestrator.runTaskLifecycle(input.task.id, async () => {
+      // A direct duplicate teardown may still be draining operations that
+      // target this task's stable compose/container names. Never recreate
+      // those names until every close caller has crossed the shared fence.
+      await orchestrator.waitForChannelClose(input.task.id);
+      // A command retry that arrives after the original start completed must
+      // not rerender/compose-up/uai-init underneath live agent sessions. Treat
+      // that lost-response retry as the successful operation it already was:
+      // every cloud caller records a non-ok taskUp as an errored task.
+      const existingTask = getHostTask(input.task.id);
+      if (existingTask?.statusMirror === "running") {
+        if (!existingTask.composeProject || !existingTask.worktreePath) {
+          return {
+            ok: false,
+            code: HostErrorCode.Internal,
+            message:
+              `task ${input.task.id} is running but its persisted runtime ` +
+              "metadata is incomplete",
+          };
+        }
+        // The mirror may be stale after a manual `docker compose down`.
+        // Confirm the app itself is live before treating this as a lost-response
+        // retry; otherwise fall through to the normal taskUp self-heal.
+        const appRuntime = await dockerCli(
+          [
+            "ps",
+            "-a",
+            "--filter",
+            `label=com.docker.compose.project=${existingTask.composeProject}`,
+            "--filter",
+            "label=com.docker.compose.service=app",
+            "--format",
+            "{{.State}}",
+          ],
+          { timeoutMs: 10_000 },
         );
+        if (appRuntime.status !== 0) {
+          return {
+            ok: false,
+            code: HostErrorCode.HostUnavailable,
+            message:
+              `could not verify runtime state for task ${input.task.id}: ` +
+              (appRuntime.stderr.trim().slice(0, 200) ||
+                "docker did not answer"),
+            retryable: true,
+          };
+        }
+        if (
+          appRuntime.stdout
+            .split(/\r?\n/)
+            .some((state) => state.trim() === "running")
+        ) {
+          // waitForChannelClose crossed the teardown fence above. A completed
+          // close-only teardown may have left this running task tombstoned, so
+          // the idempotent-success transition must reopen delivery just like a
+          // fresh successful taskUp does.
+          orchestrator.allowChannel(input.task.id);
+          return {
+            ok: true,
+            value: {
+              composeProject: existingTask.composeProject,
+              worktreePath: existingTask.worktreePath,
+              codeServerPort: existingTask.codeServerPort ?? undefined,
+              previewPorts: parsePreviewPortRuntimes(
+                existingTask.previewPorts,
+              ),
+            },
+          };
+        }
+        // Docker positively proved the mirrored app is absent/stopped. Drain
+        // and invalidate any stale in-memory channel before taskUp recreates
+        // the same stable compose/container names.
+        await orchestrator.closeChannel(input.task.id);
       }
-      return agent.taskUp(input);
-    });
-    if (result.ok) {
-      recordTaskUpResult(input.task.id, result.value);
-      recordHostEvent(input.task.id, "task.started");
-      // Degraded start (uai-init failed twice): the task is up, but say so
-      // in the feed — a silent half-start reads as a broken product.
-      if (result.value.initWarning) {
-        getOrchestrator().emitSystemNote(
-          input.task.id,
-          result.value.initWarning,
-        );
+      recordTaskStarting(input.task.id);
+      recordTaskOwner(
+        input.task.id,
+        input.task.ownerUserId,
+        input.ownerEmail ?? null,
+        input.ownerName ?? null,
+        input.projects.map((p) => p.slug),
+      );
+      // ADR-048: persist the cloud-issued per-task cli secret (host-private) so the
+      // `uai` CLI's per-agent tokens can be minted now + after a host restart.
+      if (input.task.cliSecret) {
+        storeTaskCliSecret(input.task.id, input.task.cliSecret);
       }
-      // GitHub auth for the container is best-effort (ADR-027) and runs in the
-      // background — it must never block or fail task-up. Awaiting it here would
-      // couple the command result to a network token-exchange: a slow/hung
-      // exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
-      // and mark a running task as errored. setupTaskGithub injects + schedules
-      // (or emits a system note on failure) on its own; the agents come up
-      // regardless and the token lands well before the first `gh` call.
-      void setupTaskGithub(input.task.id, input.task.ownerUserId);
-    } else {
-      recordTaskError(input.task.id);
-    }
-    return result;
+      recordHostEvent(input.task.id, "task.created");
+      const result = await wrapAgent(ctx, "taskUp", async () => {
+        // Self-heal the standard base image before task-up needs it. The
+        // boot-time build is best-effort + one-shot: a host that started before
+        // Docker was ready (common on Docker Desktop) never built
+        // uai-standard:dev, so every task-up died on a doomed registry pull
+        // (live 2026-07-22). This is idempotent — a fast inspect when present,
+        // a real build only when missing/stale (Docker is definitely up now, a
+        // task just arrived). On a build FAILURE, surface WHY to the cloud (not
+        // just the host log — the operator can't read a remote user's machine).
+        const img = await ensureStandardImage();
+        if (!img.ok) {
+          throw new AgentError(
+            "STANDARD_IMAGE_UNAVAILABLE",
+            img.error
+              ? `could not prepare the host base image uai-standard:dev — the build failed. ${toolStderrDetail(img.error)}`
+              : "could not prepare the host base image uai-standard:dev — see the host log.",
+            {},
+          );
+        }
+        return agent.taskUp(input);
+      });
+      if (result.ok) {
+        orchestrator.allowChannel(input.task.id);
+        recordTaskUpResult(input.task.id, result.value);
+        recordHostEvent(input.task.id, "task.started");
+        // Degraded start (uai-init failed twice): the task is up, but say so
+        // in the feed — a silent half-start reads as a broken product.
+        if (result.value.initWarning) {
+          orchestrator.emitSystemNote(
+            input.task.id,
+            result.value.initWarning,
+          );
+        }
+        // GitHub auth for the container is best-effort (ADR-027) and runs in the
+        // background — it must never block or fail task-up. Awaiting it here would
+        // couple the command result to a network token-exchange: a slow/hung
+        // exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
+        // and mark a running task as errored. setupTaskGithub injects + schedules
+        // (or emits a system note on failure) on its own; the agents come up
+        // regardless and the token lands well before the first `gh` call.
+        void setupTaskGithub(input.task.id, input.task.ownerUserId);
+      } else {
+        recordTaskError(input.task.id);
+      }
+      return result;
+    });
   },
 
   async taskDown(ctx, input) {
     logCommand(ctx, "taskDown", input.taskId);
-    const result = await wrapAgent(ctx, "taskDown", () => agent.taskDown(input));
-    if (result.ok) {
-      recordTaskDown(input.taskId, result.value.status);
-      recordHostEvent(input.taskId, "task.ended");
-      clearRefresh(input.taskId);
-    }
-    return result;
+    const orchestrator = getOrchestrator();
+    return orchestrator.runTaskLifecycle(input.taskId, async () => {
+      // Tombstone + close before container destruction. A concurrent cloud
+      // ensure cannot reopen the channel while teardown is awaiting Docker.
+      await orchestrator.closeChannel(input.taskId);
+      // Ad-hoc preview proxies share this task's compose network and stable
+      // names. Remove them inside the same lifecycle slot before compose down.
+      try {
+        await stopPreviewSidecars(input.taskId);
+      } catch (err) {
+        console.warn(
+          `[host-agent] task ${input.taskId}: preview cleanup failed: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+      }
+      const result = await wrapAgent(ctx, "taskDown", () =>
+        agent.taskDown(input),
+      );
+      if (result.ok) {
+        // Revoke only after destruction succeeds. On failure the task is
+        // reopened below and its on-disk MCP definitions must retain the same
+        // gateway token; deleting it early would make those exact definitions
+        // look foreign and permanently strand the recovered channel.
+        clearTaskGatewayAcl(input.taskId);
+        recordTaskDown(input.taskId, result.value.status);
+        recordHostEvent(input.taskId, "task.ended");
+        clearRefresh(input.taskId);
+      } else {
+        // Destruction failed; let a subsequent ensure restore the still-running
+        // task instead of leaving it permanently tombstoned.
+        orchestrator.allowChannel(input.taskId);
+      }
+      return result;
+    });
   },
 
   async taskStatus(ctx, taskId) {
@@ -209,14 +318,17 @@ export const hostCommands: HostCommands = {
 
   async channelTeardown(ctx, taskId) {
     logCommand(ctx, "channelTeardown", taskId);
-    try {
-      await getOrchestrator().closeChannel(taskId);
-      // ADR-057: the task's gateway routes die with the channel.
-      clearTaskGatewayAcl(taskId);
-      return ok(undefined);
-    } catch (err) {
-      return failFromUnknown(err);
-    }
+    const orchestrator = getOrchestrator();
+    return orchestrator.runTaskLifecycle(taskId, async () => {
+      try {
+        await orchestrator.closeChannel(taskId);
+        // ADR-057: the task's gateway routes die with the channel.
+        clearTaskGatewayAcl(taskId);
+        return ok(undefined);
+      } catch (err) {
+        return failFromUnknown(err);
+      }
+    });
   },
 
   async channelDeliver(ctx, taskId, agentId, text) {
@@ -238,33 +350,39 @@ export const hostCommands: HostCommands = {
 
   async previewEnsure(ctx, taskId, name, containerPort) {
     logCommand(ctx, "previewEnsure", taskId, name);
-    try {
-      const task = getHostTask(taskId);
-      if (!task) {
-        return {
-          ok: false as const,
-          code: HostErrorCode.TaskNotFound,
-          message: `no such task: ${taskId}`,
-        };
+    const orchestrator = getOrchestrator();
+    return orchestrator.runTaskLifecycle(taskId, async () => {
+      try {
+        const task = getHostTask(taskId);
+        if (!task) {
+          return {
+            ok: false as const,
+            code: HostErrorCode.TaskNotFound,
+            message: `no such task: ${taskId}`,
+          };
+        }
+        if (task.statusMirror !== "running") {
+          return ok({ hostPort: null });
+        }
+        // Task-up published port (preview enabled at launch) wins.
+        const declared = parsePreviewPortRuntimes(task.previewPorts).find(
+          (port) => port.name === name,
+        );
+        if (declared) return ok({ hostPort: declared.hostPort });
+        // Else start (or find) the node-proxy sidecar NOW — the same one the
+        // tunnel lazy-starts on first access — and hand back its host port.
+        if (!task.composeProject) return ok({ hostPort: null });
+        const hostPort = await ensurePreviewSidecar({
+          taskId,
+          composeProject: task.composeProject,
+          name,
+          containerPort,
+        });
+        return ok({ hostPort });
+      } catch (err) {
+        return failFromUnknown(err);
       }
-      // Task-up published port (preview enabled at launch) wins.
-      const declared = parsePreviewPortRuntimes(task.previewPorts).find(
-        (port) => port.name === name,
-      );
-      if (declared) return ok({ hostPort: declared.hostPort });
-      // Else start (or find) the node-proxy sidecar NOW — the same one the
-      // tunnel lazy-starts on first access — and hand back its host port.
-      if (!task.composeProject) return ok({ hostPort: null });
-      const hostPort = await ensurePreviewSidecar({
-        taskId,
-        composeProject: task.composeProject,
-        name,
-        containerPort,
-      });
-      return ok({ hostPort });
-    } catch (err) {
-      return failFromUnknown(err);
-    }
+    });
   },
 
   async channelResolvePermission(ctx, taskId, agentId, requestId, decision) {

package/src/main.ts

@@ -58,7 +58,6 @@ import { dockerCli } from "../lib/docker-exec";
 import {
   ensurePreviewSidecar,
   invalidatePreviewSidecar,
-  stopPreviewSidecars,
 } from "../lib/preview-sidecar";
 import { newId } from "../lib/ulid";
 import {
@@ -87,7 +86,7 @@ import {
   type TaskDiffInput,
   type TaskDownInput,
   type TaskLaunchInput,
-  FilesOpInput,
+  type FilesOpInput,
 } from "./protocol";
 
 // Crash reporting (ADR-071) — default-ON for installed hosts via the baked
@@ -736,12 +735,24 @@ async function resolveTunnelTarget(
     // Ad-hoc port (not published): proxy via a node-proxy sidecar that publishes
     // a 127.0.0.1 port forwarding to <app>:<containerPort> (ADR-043).
     if (frame.containerPort && task.composeProject) {
-      const hostPort = await ensurePreviewSidecar({
-        taskId: frame.taskId,
-        composeProject: task.composeProject,
-        name: frame.name,
-        containerPort: frame.containerPort,
-      });
+      const hostPort = await getOrchestrator().runTaskLifecycle(
+        frame.taskId,
+        async () => {
+          const current = getHostTask(frame.taskId);
+          if (
+            current?.statusMirror !== "running" ||
+            !current.composeProject
+          ) {
+            return null;
+          }
+          return ensurePreviewSidecar({
+            taskId: frame.taskId,
+            composeProject: current.composeProject,
+            name: frame.name!,
+            containerPort: frame.containerPort!,
+          });
+        },
+      );
       if (hostPort) return { host: "127.0.0.1", port: hostPort };
     }
   }
@@ -883,8 +894,6 @@ function dispatchCommand(
       return hostCommands.taskUp(ctx, expectTaskLaunchInput(args, 0));
     case "taskDown": {
       const downInput = expectTaskDownInput(args, 0);
-      // ADR-043: tear down this task's preview sidecars. Best-effort.
-      void stopPreviewSidecars(downInput.taskId).catch(() => {});
       return hostCommands.taskDown(ctx, downInput);
     }
     case "taskStatus":