@runuai/host 0.9.155 → 0.9.156

package/lib/compose-network-recovery.ts

@@ -0,0 +1,264 @@
+/**
+ * Bounded recovery for Docker's local-network address pool.
+ *
+ * Compose allocates one bridge network per local task. A task that fails after
+ * creating its network can remain resumable in `error`, and Docker retains the
+ * empty network. Enough such networks eventually make every later task fail
+ * with "all predefined address pools have been fully subnetted".
+ *
+ * This sweep is deliberately narrower than `docker network prune`:
+ *
+ * - the exact task/project identity must be present in trusted host state;
+ * - the task must not be starting or running;
+ * - Docker must independently report no attached container; and
+ * - deletion happens inside that task's lifecycle slot, so Resume/Stop cannot
+ *   interleave with the proof.
+ *
+ * Any unreadable or malformed observation means "do nothing", never
+ * permission to delete.
+ */
+
+import type { HostTask } from "../db/schema";
+import { dockerCli, type DockerResult } from "./docker-exec";
+import { getHostTask } from "./runtime-state";
+import { isSafeHostTaskId } from "./task-identity";
+import { isTerminal } from "./task-status";
+import { parseTaskEnvironmentLocator } from "./task-environment/types";
+
+const COMPOSE_PROJECT_LABEL = "com.docker.compose.project";
+const NETWORK_LIST_FORMAT =
+  `{{.ID}}\t{{.Name}}\t{{.Label "${COMPOSE_PROJECT_LABEL}"}}`;
+const MAX_NETWORK_LIST_BYTES = 2 * 1024 * 1024;
+const MAX_NETWORK_INSPECT_BYTES = 1024 * 1024;
+const DOCKER_TIMEOUT_MS = 10_000;
+
+interface ComposeNetworkCandidate {
+  id: string;
+  name: string;
+  composeProject: string;
+  taskId: string;
+}
+
+export interface ComposeNetworkLifecycle {
+  runTaskLifecycle<T>(
+    taskId: string,
+    operation: () => Promise<T>,
+  ): Promise<T>;
+}
+
+export interface ComposeNetworkRecoveryDeps {
+  docker(args: string[], options?: {
+    timeoutMs?: number;
+    maxOutputBytes?: number;
+  }): Promise<DockerResult>;
+  task(taskId: string): HostTask | null;
+}
+
+export interface ComposeNetworkRecoveryResult {
+  inventory: "complete" | "unavailable" | "invalid";
+  candidates: number;
+  removed: number;
+}
+
+const defaultDeps: ComposeNetworkRecoveryDeps = {
+  docker: dockerCli,
+  task: getHostTask,
+};
+
+function parseCandidate(
+  line: string,
+): ComposeNetworkCandidate | "unrelated" | null {
+  const fields = line.split("\t");
+  if (fields.length !== 3) return null;
+  const [id, name, composeProject] = fields as [string, string, string];
+  if (!/^[a-f0-9]{12,64}$/.test(id)) return null;
+  if (!composeProject.startsWith("task-")) return "unrelated";
+  const taskId = composeProject.slice("task-".length);
+  if (!isSafeHostTaskId(taskId) || taskId !== taskId.toLowerCase()) return null;
+  if (name !== `${composeProject}_default`) return null;
+  return { id, name, composeProject, taskId };
+}
+
+function parseCandidates(stdout: string): ComposeNetworkCandidate[] | null {
+  const candidates: ComposeNetworkCandidate[] = [];
+  const ids = new Set<string>();
+  for (const line of stdout.split(/\r?\n/)) {
+    if (line.length === 0) continue;
+    const candidate = parseCandidate(line);
+    if (candidate === "unrelated") continue;
+    if (candidate === null || ids.has(candidate.id)) return null;
+    ids.add(candidate.id);
+    candidates.push(candidate);
+  }
+  return candidates;
+}
+
+function taskOwnsReclaimableNetwork(
+  task: HostTask | null,
+  candidate: ComposeNetworkCandidate,
+): boolean {
+  if (
+    task === null ||
+    task.taskId !== candidate.taskId ||
+    (task.environmentProvider !== null &&
+      task.environmentProvider !== "docker-compose") ||
+    (task.statusMirror !== "error" &&
+      task.statusMirror !== "stopped" &&
+      !(typeof task.statusMirror === "string" && isTerminal(task.statusMirror)))
+  ) {
+    return false;
+  }
+  if (task.composeProject === candidate.composeProject) return true;
+  if (!task.environmentLocator) return false;
+  try {
+    const locator = parseTaskEnvironmentLocator(task.environmentLocator);
+    if (
+      locator.schemaVersion !== 1 ||
+      locator.provider !== "docker-compose" ||
+      !locator.value ||
+      typeof locator.value !== "object" ||
+      Array.isArray(locator.value)
+    ) {
+      return false;
+    }
+    const value = locator.value as Record<string, unknown>;
+    return (
+      value.schemaVersion === 2 &&
+      value.taskId === candidate.taskId &&
+      value.composeProject === candidate.composeProject &&
+      value.containerName === `${candidate.composeProject}-app-1`
+    );
+  } catch {
+    return false;
+  }
+}
+
+function inspectedEmptyNetwork(
+  stdout: string,
+  candidate: ComposeNetworkCandidate,
+): { fullId: string } | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stdout);
+  } catch {
+    return null;
+  }
+  if (!Array.isArray(parsed) || parsed.length !== 1) return null;
+  const network = parsed[0];
+  if (!network || typeof network !== "object" || Array.isArray(network)) {
+    return null;
+  }
+  const record = network as Record<string, unknown>;
+  const labels = record.Labels;
+  const containers = record.Containers;
+  if (
+    typeof record.Id !== "string" ||
+    !/^[a-f0-9]{64}$/.test(record.Id) ||
+    !record.Id.startsWith(candidate.id) ||
+    record.Name !== candidate.name ||
+    record.Driver !== "bridge" ||
+    record.Scope !== "local" ||
+    !labels ||
+    typeof labels !== "object" ||
+    Array.isArray(labels) ||
+    (labels as Record<string, unknown>)[COMPOSE_PROJECT_LABEL] !==
+      candidate.composeProject ||
+    !containers ||
+    typeof containers !== "object" ||
+    Array.isArray(containers) ||
+    Object.keys(containers).length !== 0
+  ) {
+    return null;
+  }
+  return { fullId: record.Id };
+}
+
+async function reclaimCandidate(
+  candidate: ComposeNetworkCandidate,
+  deps: ComposeNetworkRecoveryDeps,
+): Promise<boolean> {
+  if (!taskOwnsReclaimableNetwork(deps.task(candidate.taskId), candidate)) {
+    return false;
+  }
+  const inspected = await deps.docker(
+    ["network", "inspect", candidate.id],
+    {
+      timeoutMs: DOCKER_TIMEOUT_MS,
+      maxOutputBytes: MAX_NETWORK_INSPECT_BYTES,
+    },
+  );
+  if (inspected.status !== 0 || inspected.outputTruncated) return false;
+  const empty = inspectedEmptyNetwork(inspected.stdout, candidate);
+  if (empty === null) return false;
+
+  // `.Containers` is the network's own endpoint inventory. Independently ask
+  // Docker's all-container inventory as a second positive emptiness proof.
+  const attached = await deps.docker(
+    ["ps", "-aq", "--filter", `network=${empty.fullId}`],
+    {
+      timeoutMs: DOCKER_TIMEOUT_MS,
+      maxOutputBytes: MAX_NETWORK_INSPECT_BYTES,
+    },
+  );
+  if (
+    attached.status !== 0 ||
+    attached.outputTruncated ||
+    attached.stdout.trim().length !== 0
+  ) {
+    return false;
+  }
+
+  const removed = await deps.docker(["network", "rm", empty.fullId], {
+    timeoutMs: DOCKER_TIMEOUT_MS,
+    maxOutputBytes: MAX_NETWORK_INSPECT_BYTES,
+  });
+  return removed.status === 0 && !removed.outputTruncated;
+}
+
+/**
+ * Remove only positively-owned, empty task networks. This is best-effort host
+ * hygiene: an unavailable inventory never blocks a task that may still fit in
+ * Docker's pool, and it never authorizes deletion.
+ */
+export async function reclaimUnusedUaiComposeNetworks(
+  lifecycle: ComposeNetworkLifecycle,
+  deps: ComposeNetworkRecoveryDeps = defaultDeps,
+): Promise<ComposeNetworkRecoveryResult> {
+  const listed = await deps.docker(
+    [
+      "network",
+      "ls",
+      "--filter",
+      `label=${COMPOSE_PROJECT_LABEL}`,
+      "--format",
+      NETWORK_LIST_FORMAT,
+    ],
+    {
+      timeoutMs: DOCKER_TIMEOUT_MS,
+      maxOutputBytes: MAX_NETWORK_LIST_BYTES,
+    },
+  );
+  if (listed.status !== 0 || listed.outputTruncated) {
+    return { inventory: "unavailable", candidates: 0, removed: 0 };
+  }
+  const candidates = parseCandidates(listed.stdout);
+  if (candidates === null) {
+    return { inventory: "invalid", candidates: 0, removed: 0 };
+  }
+
+  let removed = 0;
+  for (const candidate of candidates) {
+    // Avoid queueing behind a live start merely to rediscover in-slot that it
+    // is not reclaimable. This observation grants no authority; the complete
+    // ownership/status check is repeated after the lifecycle slot is held.
+    if (!taskOwnsReclaimableNetwork(deps.task(candidate.taskId), candidate)) {
+      continue;
+    }
+    await lifecycle.runTaskLifecycle(candidate.taskId, async () => {
+      // The trusted row and Docker evidence are deliberately read only after
+      // entering the candidate task's lifecycle slot.
+      if (await reclaimCandidate(candidate, deps)) removed += 1;
+    });
+  }
+  return { inventory: "complete", candidates: candidates.length, removed };
+}

package/lib/standard-image.ts

@@ -714,6 +714,24 @@ export function standardImageVolumeMaintenanceCommandForTest(
     `ensure_real_dir "$node_root" node-root; ` +
     `ensure_real_dir "$shims_root" shims-root; ` +
     `ensure_real_dir "$corepack_root" corepack-root; ` +
+    // Legacy images could seed individual shims as root even though the
+    // directory itself was later repaired. asdf 0.14 then prints permission
+    // errors while reshim quietly leaves the old executable behind; task
+    // materialization can appear successful but the next CLI launch is stale
+    // or non-executable. The trusted root maintenance transaction repairs only
+    // the flat, regular-file shim set before dropping to node for every asdf
+    // mutation. Anything else is ambiguous volume content and fails closed.
+    `repair_shim_metadata() { ` +
+    `unsafe_shim_entry="$(/usr/bin/find "$shims_root" -xdev -mindepth 1 ` +
+    `-maxdepth 1 ! -type f -print -quit)"; ` +
+    `[ -z "$unsafe_shim_entry" ] || fail unsafe-shim-entry; ` +
+    `/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
+    `-exec /usr/bin/chown node:node -- {} + ` +
+    `|| fail repair-shim-owner; ` +
+    `/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
+    `-exec /usr/bin/chmod 0755 -- {} + ` +
+    `|| fail repair-shim-mode; }; ` +
+    `repair_shim_metadata; ` +
     `candidate_is_safe() { ` +
     `lexical_candidate="$(/usr/bin/realpath -m -s -- "$candidate")" ` +
     `|| return 1; [ "$lexical_candidate" = "$node_root/$default_node" ] ` +

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.9.155",
+  "version": "0.9.156",
   "description": "Uai host — runs ephemeral AI tasks in containers on a machine you control.",
   "license": "MIT",
   "author": "Uai Tech <team@runuai.com>",

package/scripts/agent/task-up.sh

@@ -2607,8 +2607,27 @@ if [ "$TASK_RUNTIME_MODE" = "apple-container" ]; then
   fi
 else
   step "COMPOSE_UP_FAILED" "docker compose up -d"
-  printf '%s\n' "$compose_env_override" | docker compose -p "$compose_project" \
-    -f "$task_uai_dir/docker-compose.yml" -f - up -d --no-build >/dev/null
+  compose_up_error=""
+  # Keep the diagnostic in memory only, but bound it: a broken plugin or
+  # daemon must not make task-up retain an arbitrarily large stderr stream.
+  # Docker's allocation error is emitted at the end of Compose's failure.
+  if ! compose_up_error=$(printf '%s\n' "$compose_env_override" \
+    | docker compose -p "$compose_project" \
+      -f "$task_uai_dir/docker-compose.yml" -f - up -d --no-build \
+      2>&1 >/dev/null | tail -c 65536); then
+    [ -z "$compose_up_error" ] || printf '%s\n' "$compose_up_error" >&2
+    require_docker_after_failed_step "docker compose up -d"
+    if printf '%s\n' "$compose_up_error" \
+      | grep -Fq 'all predefined address pools have been fully subnetted'; then
+      emit_err "COMPOSE_UP_FAILED" \
+        "Docker has no subnet left for this task. Uai already reclaimed every empty task network it could prove safe. Close an unused stopped/error task, or enlarge Docker's non-overlapping default address pools and restart Docker, then press Resume." \
+        "docker compose up -d"
+    fi
+    emit_err "COMPOSE_UP_FAILED" \
+      "Uai could not start this task's Docker Compose stack. Fix the Docker error above, then press Resume." \
+      "docker compose up -d"
+  fi
+  unset compose_up_error
 fi
 
 # Rendered YAML is not runtime evidence: Compose could reuse unexpected state,

package/src/index.ts

@@ -126,6 +126,7 @@ import {
   waitForContainerRuntimeOperational,
 } from "../lib/container-runtime";
 import { vnodeAdmissionProblem } from "../lib/vnode-pressure";
+import { reclaimUnusedUaiComposeNetworks } from "../lib/compose-network-recovery";
 import {
   machineTasksEnabled,
   provisionTaskEnvironment,
@@ -616,6 +617,41 @@ async function taskUpOn(
       );
     }
   }
+  // Every local Docker task consumes one daemon address-pool subnet. A task
+  // that failed after Compose created its network can retain an empty subnet
+  // while it waits for Resume. Reclaim only networks whose trusted host row,
+  // exact Docker identity and two empty-container inventories all agree. The
+  // sweep runs before this task's lifecycle slot so concurrent failed starts
+  // can never deadlock while trying to inspect one another.
+  if (
+    !sidecarMode() &&
+    !machineTasksEnabled() &&
+    pinnedContainerRuntimeProvider() === "docker"
+  ) {
+    try {
+      const networkRecovery = await reclaimUnusedUaiComposeNetworks(
+        orchestrator,
+      );
+      if (networkRecovery.removed > 0) {
+        console.warn(
+          `[host-agent] reclaimed ${String(networkRecovery.removed)} empty UAI task ` +
+            `network(s) before starting ${input.task.id}`,
+        );
+      } else if (networkRecovery.inventory === "invalid") {
+        console.warn(
+          "[host-agent] skipped task-network recovery because Docker returned an invalid network inventory",
+        );
+      }
+    } catch (error) {
+      // Best-effort capacity recovery must not block a task when Docker still
+      // has room. `docker compose up` remains the authoritative allocation.
+      console.warn(
+        `[host-agent] task-network recovery was unavailable: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    }
+  }
   return orchestrator.runTaskLifecycle(input.task.id, releasingHold(cloud, async () => {
     // A direct duplicate teardown may still be draining operations that
     // target this task's stable compose/container names. Never recreate