@runuai/host 0.9.155 → 0.9.157

package/lib/compose-network-recovery.ts

@@ -0,0 +1,283 @@
+/**
+ * Bounded recovery for Docker's local-network address pool.
+ *
+ * Compose allocates one bridge network per local task. A task that fails after
+ * creating its network can remain resumable in `error`, and Docker retains the
+ * empty network. Enough such networks eventually make every later task fail
+ * with "all predefined address pools have been fully subnetted".
+ *
+ * This sweep is deliberately narrower than `docker network prune`:
+ *
+ * - the exact task/project identity must be present in trusted host state;
+ * - the task must not be starting or running;
+ * - Docker must independently report no attached container; and
+ * - deletion happens inside that task's lifecycle slot, so Resume/Stop cannot
+ *   interleave with the proof. The slot is taken only if it is free at once:
+ *   the sweep never waits on a task's lifecycle.
+ *
+ * Any unreadable or malformed observation means "do nothing", never
+ * permission to delete.
+ */
+
+import type { HostTask } from "../db/schema";
+import { dockerCli, type DockerResult } from "./docker-exec";
+import { getHostTask } from "./runtime-state";
+import { isSafeHostTaskId } from "./task-identity";
+import { isTerminal } from "./task-status";
+import { parseTaskEnvironmentLocator } from "./task-environment/types";
+
+const COMPOSE_PROJECT_LABEL = "com.docker.compose.project";
+const NETWORK_LIST_FORMAT =
+  `{{.ID}}\t{{.Name}}\t{{.Label "${COMPOSE_PROJECT_LABEL}"}}`;
+const MAX_NETWORK_LIST_BYTES = 2 * 1024 * 1024;
+const MAX_NETWORK_INSPECT_BYTES = 1024 * 1024;
+const DOCKER_TIMEOUT_MS = 10_000;
+
+interface ComposeNetworkCandidate {
+  id: string;
+  name: string;
+  composeProject: string;
+  taskId: string;
+}
+
+export interface ComposeNetworkLifecycle {
+  /** Runs `operation` in the task's lifecycle slot only if that slot is free
+   *  at once (the orchestrator's `tryRunTaskLifecycle`); never waits. */
+  tryRunTaskLifecycle<T>(
+    taskId: string,
+    operation: () => Promise<T>,
+  ): Promise<{ ran: true; value: T } | { ran: false }>;
+}
+
+export interface ComposeNetworkRecoveryDeps {
+  docker(args: string[], options?: {
+    timeoutMs?: number;
+    maxOutputBytes?: number;
+  }): Promise<DockerResult>;
+  task(taskId: string): HostTask | null;
+}
+
+export interface ComposeNetworkRecoveryResult {
+  inventory: "complete" | "unavailable" | "invalid";
+  candidates: number;
+  removed: number;
+  /** Candidates left alone because their task's lifecycle slot was busy. */
+  busy: number;
+}
+
+const defaultDeps: ComposeNetworkRecoveryDeps = {
+  docker: dockerCli,
+  task: getHostTask,
+};
+
+function parseCandidate(
+  line: string,
+): ComposeNetworkCandidate | "unrelated" | null {
+  const fields = line.split("\t");
+  if (fields.length !== 3) return null;
+  const [id, name, composeProject] = fields as [string, string, string];
+  if (!/^[a-f0-9]{12,64}$/.test(id)) return null;
+  if (!composeProject.startsWith("task-")) return "unrelated";
+  const taskId = composeProject.slice("task-".length);
+  if (!isSafeHostTaskId(taskId) || taskId !== taskId.toLowerCase()) return null;
+  if (name !== `${composeProject}_default`) return null;
+  return { id, name, composeProject, taskId };
+}
+
+function parseCandidates(stdout: string): ComposeNetworkCandidate[] | null {
+  const candidates: ComposeNetworkCandidate[] = [];
+  const ids = new Set<string>();
+  for (const line of stdout.split(/\r?\n/)) {
+    if (line.length === 0) continue;
+    const candidate = parseCandidate(line);
+    if (candidate === "unrelated") continue;
+    if (candidate === null || ids.has(candidate.id)) return null;
+    ids.add(candidate.id);
+    candidates.push(candidate);
+  }
+  return candidates;
+}
+
+function taskOwnsReclaimableNetwork(
+  task: HostTask | null,
+  candidate: ComposeNetworkCandidate,
+): boolean {
+  if (
+    task === null ||
+    task.taskId !== candidate.taskId ||
+    (task.environmentProvider !== null &&
+      task.environmentProvider !== "docker-compose") ||
+    (task.statusMirror !== "error" &&
+      task.statusMirror !== "stopped" &&
+      !(typeof task.statusMirror === "string" && isTerminal(task.statusMirror)))
+  ) {
+    return false;
+  }
+  if (task.composeProject === candidate.composeProject) return true;
+  if (!task.environmentLocator) return false;
+  try {
+    const locator = parseTaskEnvironmentLocator(task.environmentLocator);
+    if (
+      locator.schemaVersion !== 1 ||
+      locator.provider !== "docker-compose" ||
+      !locator.value ||
+      typeof locator.value !== "object" ||
+      Array.isArray(locator.value)
+    ) {
+      return false;
+    }
+    const value = locator.value as Record<string, unknown>;
+    return (
+      value.schemaVersion === 2 &&
+      value.taskId === candidate.taskId &&
+      value.composeProject === candidate.composeProject &&
+      value.containerName === `${candidate.composeProject}-app-1`
+    );
+  } catch {
+    return false;
+  }
+}
+
+function inspectedEmptyNetwork(
+  stdout: string,
+  candidate: ComposeNetworkCandidate,
+): { fullId: string } | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stdout);
+  } catch {
+    return null;
+  }
+  if (!Array.isArray(parsed) || parsed.length !== 1) return null;
+  const network = parsed[0];
+  if (!network || typeof network !== "object" || Array.isArray(network)) {
+    return null;
+  }
+  const record = network as Record<string, unknown>;
+  const labels = record.Labels;
+  const containers = record.Containers;
+  if (
+    typeof record.Id !== "string" ||
+    !/^[a-f0-9]{64}$/.test(record.Id) ||
+    !record.Id.startsWith(candidate.id) ||
+    record.Name !== candidate.name ||
+    record.Driver !== "bridge" ||
+    record.Scope !== "local" ||
+    !labels ||
+    typeof labels !== "object" ||
+    Array.isArray(labels) ||
+    (labels as Record<string, unknown>)[COMPOSE_PROJECT_LABEL] !==
+      candidate.composeProject ||
+    !containers ||
+    typeof containers !== "object" ||
+    Array.isArray(containers) ||
+    Object.keys(containers).length !== 0
+  ) {
+    return null;
+  }
+  return { fullId: record.Id };
+}
+
+async function reclaimCandidate(
+  candidate: ComposeNetworkCandidate,
+  deps: ComposeNetworkRecoveryDeps,
+): Promise<boolean> {
+  if (!taskOwnsReclaimableNetwork(deps.task(candidate.taskId), candidate)) {
+    return false;
+  }
+  const inspected = await deps.docker(
+    ["network", "inspect", candidate.id],
+    {
+      timeoutMs: DOCKER_TIMEOUT_MS,
+      maxOutputBytes: MAX_NETWORK_INSPECT_BYTES,
+    },
+  );
+  if (inspected.status !== 0 || inspected.outputTruncated) return false;
+  const empty = inspectedEmptyNetwork(inspected.stdout, candidate);
+  if (empty === null) return false;
+
+  // `.Containers` is the network's own endpoint inventory. Independently ask
+  // Docker's all-container inventory as a second positive emptiness proof.
+  const attached = await deps.docker(
+    ["ps", "-aq", "--filter", `network=${empty.fullId}`],
+    {
+      timeoutMs: DOCKER_TIMEOUT_MS,
+      maxOutputBytes: MAX_NETWORK_INSPECT_BYTES,
+    },
+  );
+  if (
+    attached.status !== 0 ||
+    attached.outputTruncated ||
+    attached.stdout.trim().length !== 0
+  ) {
+    return false;
+  }
+
+  const removed = await deps.docker(["network", "rm", empty.fullId], {
+    timeoutMs: DOCKER_TIMEOUT_MS,
+    maxOutputBytes: MAX_NETWORK_INSPECT_BYTES,
+  });
+  return removed.status === 0 && !removed.outputTruncated;
+}
+
+/**
+ * Remove only positively-owned, empty task networks. This is best-effort host
+ * hygiene: an unavailable inventory never blocks a task that may still fit in
+ * Docker's pool, and it never authorizes deletion.
+ */
+export async function reclaimUnusedUaiComposeNetworks(
+  lifecycle: ComposeNetworkLifecycle,
+  deps: ComposeNetworkRecoveryDeps = defaultDeps,
+): Promise<ComposeNetworkRecoveryResult> {
+  const listed = await deps.docker(
+    [
+      "network",
+      "ls",
+      "--filter",
+      `label=${COMPOSE_PROJECT_LABEL}`,
+      "--format",
+      NETWORK_LIST_FORMAT,
+    ],
+    {
+      timeoutMs: DOCKER_TIMEOUT_MS,
+      maxOutputBytes: MAX_NETWORK_LIST_BYTES,
+    },
+  );
+  if (listed.status !== 0 || listed.outputTruncated) {
+    return { inventory: "unavailable", candidates: 0, removed: 0, busy: 0 };
+  }
+  const candidates = parseCandidates(listed.stdout);
+  if (candidates === null) {
+    return { inventory: "invalid", candidates: 0, removed: 0, busy: 0 };
+  }
+
+  let removed = 0;
+  let busy = 0;
+  for (const candidate of candidates) {
+    // Avoid taking a live start's slot merely to rediscover in-slot that it
+    // is not reclaimable. This observation grants no authority; the complete
+    // ownership/status check is repeated after the lifecycle slot is held.
+    if (!taskOwnsReclaimableNetwork(deps.task(candidate.taskId), candidate)) {
+      continue;
+    }
+    // Never wait for the candidate's slot. A Resume marks its task `starting`
+    // only partway into its own slot, so the row can still read `error` while
+    // that slot is held, and waiting for it meant waiting out the whole
+    // start; a stuck operation on any stopped task would hold every later
+    // start here. A busy task's network is left for a later start's sweep.
+    const attempt = await lifecycle.tryRunTaskLifecycle(
+      candidate.taskId,
+      // The trusted row and Docker evidence are deliberately read only after
+      // entering the candidate task's lifecycle slot.
+      () => reclaimCandidate(candidate, deps),
+    );
+    if (!attempt.ran) busy += 1;
+    else if (attempt.value) removed += 1;
+  }
+  return {
+    inventory: "complete",
+    candidates: candidates.length,
+    removed,
+    busy,
+  };
+}

package/lib/credentials-activation-authority.ts

@@ -114,7 +114,7 @@ export type ActivationAnswer = { ok: true } | { ok: false; error: string };
 const RECORD_SCHEMA_VERSION = 1;
 const STEPS: readonly ActivationStep[] = ["local", "cloud", "active"];
 
-export interface ActivationAuthorityDeps {
+export interface ActivationAuthorityDeps<Retirement = RetirementPlan> {
   /** The durable record (non-secret), beside the runner's database. */
   recordPath: string;
   /** The credentials floor's file exists: the switch began, even with no
@@ -122,13 +122,17 @@ export interface ActivationAuthorityDeps {
   floorClaimed: () => boolean;
   /** Whether this runner moves to cloud credentials in this release. */
   eligible: () => boolean;
+  /** A marked task machine is born behind the v1 fence: it may ask the
+   *  cloud to activate, but must never expose a transient `legacy` mode or
+   *  request a v0 lease while that answer is pending. */
+  initiallyPending?: () => boolean;
   /** One phase asked of the cloud. Rejects when it could not be asked or
    *  answered (no bridge, a timeout): the next attempt asks again. */
   ask: (phase: CredentialsActivationPhase) => Promise<ActivationAnswer>;
   /** Admit no new start on the runner's own credentials (cloud-credential-host). */
   pauseLegacyStarts: () => { drained: Promise<void>; resume(): void };
   /** The retirement's plan: changes nothing, rejects with every blocker. */
-  planRetirement: () => Promise<RetirementPlan>;
+  planRetirement: () => Promise<Retirement>;
   /** Under the managed-operation lock, prove the next boot reads the record
    *  and fence standalone recovery, then call `record` to write it
    *  (managed-runtime.ts commitCredentialsSwitchLocally). */
@@ -140,7 +144,7 @@ export interface ActivationAuthorityDeps {
   /** Carry the plan out (idempotent). `stopped` names each task whose
    *  runtime is gone, at its exact run epoch; `problem` says what could not
    *  be proven removed. */
-  retire: (plan: RetirementPlan) => Promise<{ stopped: StoppedReport[]; problem: string | null }>;
+  retire: (plan: Retirement) => Promise<{ stopped: StoppedReport[]; problem: string | null }>;
   /** Send each owed report again; returns those still owed (the moot ones
    *  dropped: a newer run of the task, or its row gone). */
   reportStopped: (owed: readonly StoppedReport[]) => StoppedReport[];
@@ -165,8 +169,8 @@ export interface ActivationAuthorityDeps {
   log: (line: string) => void;
 }
 
-export class CredentialsActivationAuthority {
-  readonly #deps: ActivationAuthorityDeps;
+export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
+  readonly #deps: ActivationAuthorityDeps<Retirement>;
   #record: ActivationRecord | null;
   /** `#record` is known to be durable (see the header). */
   #durable = false;
@@ -179,13 +183,18 @@ export class CredentialsActivationAuthority {
   #sweepsItself = false;
   /** The record was `active` when this process started. */
   readonly #activeAtBoot: boolean;
+  /** A fresh task-machine incarnation starts behind the v1 fence before it
+   *  has a local activation record. Unlike an owned runner's reversible
+   *  prepare, a cloud-marked incarnation must never fall back to v0. */
+  readonly #initiallyPending: boolean;
   #advancing: Promise<void> | null = null;
   #lastNote: string | null = null;
   /** Leaving the runner's own credentials failed in this process. */
   #stuck = false;
 
-  constructor(deps: ActivationAuthorityDeps) {
+  constructor(deps: ActivationAuthorityDeps<Retirement>) {
     this.#deps = deps;
+    this.#initiallyPending = deps.initiallyPending?.() === true;
     this.#record = readActivationRecord(deps.recordPath);
     if (this.#record === null && deps.floorClaimed()) {
       // Only the switch claims the floor, after its record: a floor with no
@@ -210,7 +219,9 @@ export class CredentialsActivationAuthority {
   }
 
   mode(): CredentialsMode {
-    if (this.#record === null) return "legacy";
+    if (this.#record === null) {
+      return this.#initiallyPending ? "pending" : "legacy";
+    }
     return this.#record.step === "active" && (this.#announced || this.#announcing) ? "active" : "pending";
   }
 
@@ -253,7 +264,7 @@ export class CredentialsActivationAuthority {
       await this.#activate();
       return;
     }
-    let plan: RetirementPlan | null = null;
+    let plan: Retirement | null = null;
     if (this.#record === null) {
       if (!this.#deps.eligible()) return;
       const prepared = await this.#deps.ask("prepare");
@@ -276,7 +287,11 @@ export class CredentialsActivationAuthority {
         this.#record ??= readActivationRecord(this.#deps.recordPath);
         if (this.#record === null) {
           paused.resume();
-          this.#note(`the runner stays on its own credentials, nothing changed: ${message(error)}`);
+          this.#note(
+            this.#initiallyPending
+              ? `the v1-required task machine stays pending, with no v0 fallback: ${message(error)}`
+              : `the runner stays on its own credentials, nothing changed: ${message(error)}`,
+          );
           return;
         }
       }
@@ -395,7 +410,7 @@ export class CredentialsActivationAuthority {
 
   /** The retirement, its reports owed durably before they are sent, and
    *  sent; rejects when anything could not be proven removed. */
-  async #retire(plan: RetirementPlan): Promise<void> {
+  async #retire(plan: Retirement): Promise<void> {
     const { stopped, problem } = await this.#deps.retire(plan);
     const owed = new Map((this.#record?.stopped ?? []).map((report) => [report.taskId, report]));
     for (const report of stopped) owed.set(report.taskId, report);

package/lib/orchestrator.ts

@@ -847,6 +847,35 @@ export class Orchestrator {
     }
   }
 
+  /** Run one task lifecycle mutation only if the task's slot is free right
+   *  now: nothing running or queued for it, and its boot recovery finished.
+   *  Never waits, so best-effort work on another task's resources (the
+   *  task-network reclaim) cannot be held up by a long start or a stuck
+   *  operation. The check and the claim happen in one synchronous step, so
+   *  nothing can enter between them, and a `runTaskLifecycle` that arrives
+   *  meanwhile queues behind this one as it would behind any other. */
+  async tryRunTaskLifecycle<T>(
+    taskId: string,
+    operation: () => Promise<T>,
+  ): Promise<{ ran: true; value: T } | { ran: false }> {
+    if (this.taskLifecycleTails.has(taskId) || taskRecoveryPending(taskId)) {
+      return { ran: false };
+    }
+    let release!: () => void;
+    const tail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    this.taskLifecycleTails.set(taskId, tail);
+    try {
+      return { ran: true, value: await operation() };
+    } finally {
+      release();
+      if (this.taskLifecycleTails.get(taskId) === tail) {
+        this.taskLifecycleTails.delete(taskId);
+      }
+    }
+  }
+
   /** Emit a host-originated system note into a task's channel (ADR-027). */
   emitSystemNote(taskId: string, text: string): void {
     this.emitHost({ kind: "system.note", taskId, text });
@@ -7481,6 +7510,12 @@ export function taskRecoveryComplete(taskId: string): Promise<void> {
   return taskRecoveryBarriers.get(taskId)?.promise ?? Promise.resolve();
 }
 
+/** Boot recovery has not yet finished with this task. Work that must not
+ *  wait (`tryRunTaskLifecycle`) skips the task rather than join its barrier. */
+export function taskRecoveryPending(taskId: string): boolean {
+  return taskRecoveryBarriers.has(taskId);
+}
+
 /**
  * Resolves once boot-time recovery has finished (starting the orchestrator —
  * and with it recovery — if that hasn't happened yet). Never rejects; false

package/lib/standard-image.ts

@@ -714,6 +714,24 @@ export function standardImageVolumeMaintenanceCommandForTest(
     `ensure_real_dir "$node_root" node-root; ` +
     `ensure_real_dir "$shims_root" shims-root; ` +
     `ensure_real_dir "$corepack_root" corepack-root; ` +
+    // Legacy images could seed individual shims as root even though the
+    // directory itself was later repaired. asdf 0.14 then prints permission
+    // errors while reshim quietly leaves the old executable behind; task
+    // materialization can appear successful but the next CLI launch is stale
+    // or non-executable. The trusted root maintenance transaction repairs only
+    // the flat, regular-file shim set before dropping to node for every asdf
+    // mutation. Anything else is ambiguous volume content and fails closed.
+    `repair_shim_metadata() { ` +
+    `unsafe_shim_entry="$(/usr/bin/find "$shims_root" -xdev -mindepth 1 ` +
+    `-maxdepth 1 ! -type f -print -quit)"; ` +
+    `[ -z "$unsafe_shim_entry" ] || fail unsafe-shim-entry; ` +
+    `/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
+    `-exec /usr/bin/chown node:node -- {} + ` +
+    `|| fail repair-shim-owner; ` +
+    `/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
+    `-exec /usr/bin/chmod 0755 -- {} + ` +
+    `|| fail repair-shim-mode; }; ` +
+    `repair_shim_metadata; ` +
     `candidate_is_safe() { ` +
     `lexical_candidate="$(/usr/bin/realpath -m -s -- "$candidate")" ` +
     `|| return 1; [ "$lexical_candidate" = "$node_root/$default_node" ] ` +

package/lib/task-environment/self.ts

@@ -61,9 +61,23 @@ import {
 } from "./machine-environment";
 import { SELF_TASK_ENVIRONMENT_PROVIDER, SELF_TASK_USER, TASK_HOSTS_VIEW_PATH } from "./self-identity";
 import { readBoundedFile, workspacePathParts } from "./workspace-files";
+import { assertTaskFilesMachineProcessAllowed } from "../task-files-machine-retirement";
 
 export { ETC_ENVIRONMENT_PATH, parseEtcEnvironment, SELF_TASK_ENVIRONMENT_PROVIDER, SELF_TASK_USER, TASK_HOSTS_VIEW_PATH };
 
+/**
+ * A terminal Files failure latches before its synchronous handler runs. Check
+ * on both sides of a caller's own admission callback so that callback cannot
+ * trigger retirement and then let this launch repopulate the cgroup the
+ * handler just emptied. JavaScript runs the final check and node spawn in one
+ * turn, so no other callback can interleave between them.
+ */
+function admitTaskMachineCall(admit?: () => void): void {
+  assertTaskFilesMachineProcessAllowed();
+  admit?.();
+  assertTaskFilesMachineProcessAllowed();
+}
+
 /**
  * Binds the view, then runs the rest of its argv. With no view yet (nothing
  * pinned so far: a start whose CA failed, the proxy off), it first makes one,
@@ -281,7 +295,7 @@ class SelfTaskEnvironmentHandle implements TaskEnvironmentHandle {
 
   async readWorkspaceFile(path: string, options?: TaskEnvironmentReadOptions): Promise<Uint8Array | null> {
     assertWorkspaceScopedPath(this.descriptor, path);
-    options?.admit?.();
+    admitTaskMachineCall(options?.admit);
     return readBoundedFile(this.descriptor.workspacePath, workspacePathParts(this.descriptor, path), options);
   }
 
@@ -355,7 +369,7 @@ class SelfTaskEnvironmentHandle implements TaskEnvironmentHandle {
     refusePrivilegedExec(request);
     const scoped = this.#withMachineEnv(request);
     const { command, args, env } = this.#command(scoped, "interactive");
-    request.admit?.();
+    admitTaskMachineCall(request.admit);
     return capturedCliExec(this.#spawn, command, args, scoped, env);
   }
 
@@ -363,7 +377,7 @@ class SelfTaskEnvironmentHandle implements TaskEnvironmentHandle {
     assertTaskEnvironmentProcessRequest(request);
     const scoped = this.#withMachineEnv(request);
     const { command, args, env } = this.#command(scoped, "interactive");
-    request.admit?.();
+    admitTaskMachineCall(request.admit);
     return streamingCliExec(this.#spawn, command, args, scoped, env);
   }
 
@@ -374,7 +388,7 @@ class SelfTaskEnvironmentHandle implements TaskEnvironmentHandle {
       cwd: request.cwd ?? this.#value.workspacePath,
     });
     const { command, args, env } = this.#command(session, "interactive");
-    request.admit?.();
+    admitTaskMachineCall(request.admit);
     return streamingCliExec(this.#spawn, command, args, session, env);
   }
 
@@ -387,7 +401,7 @@ class SelfTaskEnvironmentHandle implements TaskEnvironmentHandle {
       cwd: request.cwd ?? this.#value.workspacePath,
     });
     const { command, args, env } = this.#command(session, "detached");
-    request.admit?.();
+    admitTaskMachineCall(request.admit);
     return capturedCliExec(
       this.#spawn,
       command,