@runuai/host 0.9.156 → 0.9.158

package/src/index.ts

@@ -346,7 +346,7 @@ function untilLeavingLegacy<T>(legacy: LegacyStartPermit | null, wait: Promise<T
 
 /** A start the runner does not take while it switches its tasks to cloud
  *  credentials: the cloud starts it again shortly. */
-function switchingToCloudCredentials(): TaskUpOutcome {
+function switchingToCloudCredentials(): HostCommandResult<never> {
   return {
     ok: false,
     code: HostErrorCode.HostUnavailable,
@@ -621,8 +621,9 @@ async function taskUpOn(
   // that failed after Compose created its network can retain an empty subnet
   // while it waits for Resume. Reclaim only networks whose trusted host row,
   // exact Docker identity and two empty-container inventories all agree. The
-  // sweep runs before this task's lifecycle slot so concurrent failed starts
-  // can never deadlock while trying to inspect one another.
+  // sweep runs before this task's lifecycle slot and never waits on another
+  // task's: a candidate whose slot is busy is skipped, so concurrent starts
+  // neither deadlock nor queue behind one another's provisioning.
   if (
     !sidecarMode() &&
     !machineTasksEnabled() &&
@@ -642,6 +643,12 @@ async function taskUpOn(
           "[host-agent] skipped task-network recovery because Docker returned an invalid network inventory",
         );
       }
+      if (networkRecovery.busy > 0) {
+        console.log(
+          `[host-agent] task-network recovery left ${String(networkRecovery.busy)} ` +
+            `network(s) alone while their task was busy`,
+        );
+      }
     } catch (error) {
       // Best-effort capacity recovery must not block a task when Docker still
       // has room. `docker compose up` remains the authoritative allocation.
@@ -1976,11 +1983,17 @@ export const hostCommands: HostCommands = {
 
   async channelEnsure(ctx, input) {
     logCommand(ctx, "channelEnsure", input.taskId);
+    if (credentialsMode() === "pending") {
+      return switchingToCloudCredentials();
+    }
     const admissionFailure = managedMaintenanceUnavailable();
     if (admissionFailure) return admissionFailure;
     const runtimeFailure = runtimeUnavailable();
     if (runtimeFailure) return runtimeFailure;
     await agentClisReady;
+    if (credentialsMode() === "pending") {
+      return switchingToCloudCredentials();
+    }
     const readyAdmissionFailure = managedMaintenanceUnavailable();
     if (readyAdmissionFailure) return readyAdmissionFailure;
     const readyRuntimeFailure = runtimeUnavailable();
@@ -2025,6 +2038,9 @@ export const hostCommands: HostCommands = {
 
   async channelDeliver(ctx, taskId, agentId, text, origin) {
     logCommand(ctx, "channelDeliver", taskId, agentId);
+    if (credentialsMode() === "pending") {
+      return switchingToCloudCredentials();
+    }
     const runtimeFailure = runtimeUnavailable();
     if (runtimeFailure) return runtimeFailure;
     try {

package/src/main.ts

@@ -302,6 +302,7 @@ import {
   LIVE_ROSTER_EDIT_PROTOCOL_FEATURE,
   AGENT_TOKEN_GENERATION_PROTOCOL_FEATURE,
   isEngineAccountScope,
+  isTaskMachineRetireAckFrame,
   HOST_LOGS_PROTOCOL_FEATURE,
   HOST_TASK_INVENTORY_PROTOCOL_FEATURE,
   MAX_ENGINE_ACCOUNT_LABEL_CHARS,
@@ -413,11 +414,29 @@ import {
 import { taskContainerBackend, taskContainerCli } from "../lib/task-container-cli";
 import { CloudCredentialRuntime } from "../lib/cloud-credential-runtime";
 import { createTaskFilesV1Consumer } from "../lib/credential-consumers/files";
-import { taskFilesV1Readiness } from "../lib/task-files-readiness";
 import {
+  TASK_FILES_SESSION_ROOT,
+  taskFilesV1Readiness,
+} from "../lib/task-files-readiness";
+import {
+  latchDamagedTaskFilesMachineRetirement,
   requestTaskFilesMachineRetirement,
+  setTaskFilesMachineRetirementHandler,
   taskFilesMachineRetirementReady,
 } from "../lib/task-files-machine-retirement";
+import {
+  kernelBootId,
+  TaskFilesMachineRetirementOutbox,
+  TASK_FILES_MACHINE_RETIREMENT_RECORD,
+  taskMachinePeerProcessIds,
+} from "../lib/task-files-machine-retirement-outbox";
+import {
+  planFreshTaskMachineCredentialsActivation,
+  taskMachineCredentialsV1Required,
+  verifyFreshTaskMachineCredentialsActivation,
+  type TaskMachineCredentialsActivationDeps,
+  type TaskMachineCredentialsActivationPlan,
+} from "../lib/task-machine-credentials-activation";
 import { ProjectEnvTaskReplicaManager } from "../lib/project-env-task-replica";
 import {
   purgeDefaultEngineSlots,
@@ -669,27 +688,82 @@ try {
   );
 }
 migrateHostDb();
+const taskMachineCredentialsRequired =
+  sidecarMode() && taskMachineCredentialsV1Required();
+const taskFilesReadiness = taskFilesV1Readiness(sidecarMode());
+const taskFilesMachineRetirementOutbox = sidecarMode()
+  ? new TaskFilesMachineRetirementOutbox({
+      // Root-only tmpfs survives runner restarts but not a machine reboot,
+      // exactly matching the retirement proof boundary. It also keeps data
+      // disk exhaustion from blocking the first durable handoff.
+      recordPath: join(
+        TASK_FILES_SESSION_ROOT,
+        TASK_FILES_MACHINE_RETIREMENT_RECORD,
+      ),
+      send: sendCredentialFrame,
+      log: (line) => console.warn(line),
+    })
+  : null;
+if (taskFilesMachineRetirementOutbox !== null) {
+  setTaskFilesMachineRetirementHandler((request) => {
+    try {
+      return taskFilesMachineRetirementOutbox.handler(request);
+    } finally {
+      // The terminal latch closes before the handler runs. Withdraw Files-v1
+      // even if persisting the outbox record fails and the same exact handoff
+      // must be retried.
+      sendCapabilities();
+    }
+  });
+  if (taskFilesMachineRetirementOutbox.startup.kind === "pending") {
+    requestTaskFilesMachineRetirement(
+      taskFilesMachineRetirementOutbox.startup.request,
+    );
+  } else if (taskFilesMachineRetirementOutbox.startup.kind === "damaged") {
+    latchDamagedTaskFilesMachineRetirement(
+      taskFilesMachineRetirementOutbox.startup.error,
+    );
+    taskFilesMachineRetirementOutbox.quarantineDamagedStartup();
+  }
+}
 // ADR-154 §13: the activation authority, before anything reads the mode
 // (the refreshers, the v0 lease, the runtime's recovery, the first
-// capability frame). It moves only owned runners in this release: a task
-// machine needs its run's Files session on v1 first, and the sign-in box
-// runs no task.
-const credentialsActivation = new CredentialsActivationAuthority({
+// capability frame). A cloud-marked task machine starts pending here—before
+// a v0 lease can be requested—and activates only after its local Files and
+// exact-machine retirement boundaries are installed.
+type CredentialsRetirementPlan =
+  | { kind: "owned-runner"; plan: RetirementPlan }
+  | TaskMachineCredentialsActivationPlan;
+const credentialsActivation = new CredentialsActivationAuthority<CredentialsRetirementPlan>({
   recordPath: join(env.dataDir, CREDENTIALS_ACTIVATION_RECORD),
   floorClaimed: () => credentialsCapabilityFloorExists(),
-  eligible: () => !sidecarMode() && !signInBoxMode(),
+  eligible: () =>
+    taskMachineCredentialsRequired
+      ? sidecarTaskId() !== null
+      : !sidecarMode() && !signInBoxMode(),
+  initiallyPending: () => taskMachineCredentialsRequired,
   ask: askCredentialsActivation,
   pauseLegacyStarts,
-  planRetirement: () => planLegacyTaskRuntimeRetirement(retirementPlanDeps()),
+  planRetirement: planCredentialsRetirement,
   commitLocally: (record) => commitCredentialsSwitchLocally(record),
-  retire: retireTaskRuntimesForCloudCredentials,
+  retire: retireCredentialsPlan,
   reportStopped: reportRetiredTasksStopped,
   claimFloor: claimCredentialsCapabilityFloor,
   leaveLegacy,
   purge: purgeCredentialStores,
   swept: cloudCredentialsSwept,
   sweep: recoverCloudCredentials,
-  onActive: cloudCredentialsActive,
+  onActive: (how) => {
+    if (
+      sidecarMode() &&
+      (!taskFilesReadiness.ready || !taskFilesMachineRetirementReady())
+    ) {
+      throw new Error(
+        "the task machine cannot activate cloud credentials without its Files and retirement boundaries",
+      );
+    }
+    cloudCredentialsActive(how);
+  },
   // Pending again: the next capability frame advertises no v1.
   onActiveWithdrawn: () => sendCapabilities(),
   restart: (why) => {
@@ -699,9 +773,13 @@ const credentialsActivation = new CredentialsActivationAuthority({
   log: (line) => console.warn(line),
 });
 setCredentialsModeSource(() => credentialsActivation.mode());
-/** ADR-154 §13: how often a runner not yet on cloud credentials asks again,
- *  so the operator turning them on reaches it within minutes. */
+/** ADR-154 §13: the steady/operator cadence. Active runners report their
+ * purge again here too, so keep it deliberately quiet. */
 const CREDENTIALS_ACTIVATION_RETRY_MS = 5 * 60_000;
+/** A newly assigned marked machine is already behind its floor. Retry a
+ * transient freshness/Files proof inside Cloud's boot budget, without making
+ * its eventual active purge report a five-second heartbeat. */
+const TASK_MACHINE_CREDENTIALS_ACTIVATION_RETRY_MS = 5_000;
 const CREDENTIALS_ACTIVATION_ANSWER_MS = 60_000;
 // Started here, first fired long after the bridge's state exists.
 const credentialsActivationTimer = setInterval(() => {
@@ -714,6 +792,18 @@ const credentialsActivationTimer = setInterval(() => {
   void credentialsActivation.advance();
 }, CREDENTIALS_ACTIVATION_RETRY_MS);
 credentialsActivationTimer.unref?.();
+const taskMachineCredentialsActivationTimer =
+  taskMachineCredentialsRequired && sidecarTaskId() !== null
+    ? setInterval(() => {
+        if (
+          stopping ||
+          !credentialFramesReady ||
+          credentialsMode() !== "pending"
+        ) return;
+        void credentialsActivation.advance();
+      }, TASK_MACHINE_CREDENTIALS_ACTIVATION_RETRY_MS)
+    : null;
+taskMachineCredentialsActivationTimer?.unref?.();
 
 const credentialsActivationAsks = new Map<
   string,
@@ -914,7 +1004,6 @@ if (signInBoxMode()) {
 // ADR-154: the composer for tasks on cloud credentials. Nothing runs on it
 // until the runner's activation authority makes cloud credentials its
 // tasks' only ones (lib/cloud-credential-host.ts).
-const taskFilesReadiness = taskFilesV1Readiness(sidecarMode());
 const taskFiles = sidecarMode()
   ? createTaskFilesV1Consumer({
       recoveredBinding: (assignment) => {
@@ -979,8 +1068,12 @@ if (sidecarMode()) {
 }
 setCloudCredentialRuntime(cloudCredentials);
 // A runner that was already on cloud credentials follows them from here:
-// once its boot recovery sweep (the container runtime's own pass) is done.
-credentialsActivation.booted();
+// once its boot recovery sweep is done. Owned runners get that sweep from
+// container-runtime activation; a task machine has no container runtime, so
+// its authority must run the composer sweep itself before it can reattach a
+// surviving agent or ask for a new grant.
+if (sidecarMode()) credentialsActivation.sweepItself();
+else credentialsActivation.booted();
 void cloudCredentialsRecovered().then(() => {
   if (!stopping) void credentialsActivation.advance();
 });
@@ -1641,6 +1734,7 @@ function sendCredentialFrame(frame: Parameters<typeof send>[1]): boolean {
  */
 function bridgeResumed(): void {
   credentialFramesReady = true;
+  taskFilesMachineRetirementOutbox?.bridgeAuthenticated();
   const mode = credentialsMode();
   if (mode === "legacy") leaseManager?.reconnected();
   else if (mode === "active") {
@@ -1688,6 +1782,47 @@ function reportCredentialsPurged(): void {
 /** ADR-154 §13: what the retirement reads (lib/legacy-runtime-retirement.ts):
  *  the task rows once, the identity fence over those same rows, the pinned
  *  engine. */
+async function planCredentialsRetirement(): Promise<CredentialsRetirementPlan> {
+  if (sidecarMode()) {
+    return planFreshTaskMachineCredentialsActivation(
+      taskMachineCredentialsActivationDeps(),
+    );
+  }
+  return {
+    kind: "owned-runner",
+    plan: await planLegacyTaskRuntimeRetirement(retirementPlanDeps()),
+  };
+}
+
+async function retireCredentialsPlan(
+  plan: CredentialsRetirementPlan,
+): Promise<{ stopped: StoppedReport[]; problem: string | null }> {
+  if (plan.kind === "owned-runner") {
+    return retireTaskRuntimesForCloudCredentials(plan.plan);
+  }
+  verifyFreshTaskMachineCredentialsActivation(
+    plan,
+    taskMachineCredentialsActivationDeps(),
+  );
+  return { stopped: [], problem: null };
+}
+
+function taskMachineCredentialsActivationDeps(): TaskMachineCredentialsActivationDeps {
+  return {
+    required: () => taskMachineCredentialsRequired,
+    taskId: sidecarTaskId,
+    machineBootId: kernelBootId,
+    filesReady: () => taskFilesReadiness.ready,
+    retirementReady: taskFilesMachineRetirementReady,
+    taskRows: () =>
+      getDb()
+        .select({ taskId: schema.hostTasks.taskId })
+        .from(schema.hostTasks)
+        .all(),
+    peerProcessIds: taskMachinePeerProcessIds,
+  };
+}
+
 function retirementPlanDeps(): RetirementPlanDeps {
   return {
     snapshot: () => {
@@ -2890,6 +3025,9 @@ async function connect(): Promise<void> {
         eventOutbox.ack(frame.seq);
         flushEvents();
         break;
+      case "task.machine.retire.ack":
+        taskFilesMachineRetirementOutbox?.acknowledge(frame);
+        break;
       case "command":
         void handleCommand(socket, frame);
         break;
@@ -4632,6 +4770,7 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
   if (frame.kind === "event.ack" && typeof frame.seq === "number") {
     return { kind: "event.ack", seq: frame.seq };
   }
+  if (isTaskMachineRetireAckFrame(frame)) return frame;
   if (
     frame.kind === "host.inventory.request" &&
     isSafeInventoryScanId(frame.scanId) &&
@@ -5203,6 +5342,7 @@ function rejectFrameDuringShutdown(
     case "credential.changed":
     case "credential.refresh.result":
     case "credentials.activate.result":
+    case "task.machine.retire.ack":
       return false;
   }
 }
@@ -5802,6 +5942,7 @@ function requestShutdown(signal: NodeJS.Signals): void {
   void stopRemoteOperationManagers();
   stopMcpGatewaySubscription();
   stopAgentClisReadySubscription();
+  taskFilesMachineRetirementOutbox?.stop();
   // ADR-154: no credential is served or taken after this; what was accepted
   // drains, bounded, and no cleanup starts (the next process's recovery runs
   // it). Exit waits for it.

package/src/protocol.ts

@@ -2961,6 +2961,26 @@ export const MAX_REMOTE_HOST_MESSAGE_BYTES = 512;
 export const MAX_HOST_TASK_INVENTORY_PAGE_SIZE = 256;
 export const MAX_HOST_TASK_INVENTORY_SCAN_ID_CHARS = 128;
 export const MAX_HOST_TASK_INVENTORY_TASK_ID_CHARS = 128;
+/** A refusal/staleness explanation is diagnostic only and must stay bounded
+ * on the authenticated control socket. */
+export const MAX_TASK_MACHINE_RETIRE_ERROR_BYTES = 512;
+/** Files-v1 terminal failures are the only reasons a task sidecar may ask the
+ * cloud to cold-stop its exact machine incarnation. Keep this vocabulary
+ * closed: the cloud persists it as audit provenance. */
+export const TASK_MACHINE_RETIRE_REASONS = [
+  "setup-unproven",
+  "visibility-unproven",
+  "mount-retirement-unproven",
+] as const;
+export type TaskMachineRetireReason =
+  (typeof TASK_MACHINE_RETIRE_REASONS)[number];
+export const TASK_MACHINE_RETIRE_OUTCOMES = [
+  "accepted",
+  "stale",
+  "refused",
+] as const;
+export type TaskMachineRetireOutcome =
+  (typeof TASK_MACHINE_RETIRE_OUTCOMES)[number];
 /**
  * A redeem response carrying this exact header proves the cloud checks a spent
  * token before expiry. That lets a new CLI distinguish a definitive 404/410
@@ -3561,6 +3581,65 @@ export function isHostOpId(value: unknown): value is HostOpId {
   );
 }
 
+function isTaskMachineRetireTaskId(value: unknown): value is string {
+  return (
+    typeof value === "string" &&
+    value.length > 0 &&
+    value.length <= MAX_HOST_TASK_INVENTORY_TASK_ID_CHARS &&
+    /^[a-z0-9][a-z0-9_-]*$/.test(value)
+  );
+}
+
+function isTaskMachineRetireRunEpoch(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) > 0;
+}
+
+export function isTaskMachineRetireFrame(
+  value: unknown,
+): value is TaskMachineRetireFrame {
+  const frame = exactWireObject(value, [
+    "kind",
+    "opId",
+    "taskId",
+    "runEpoch",
+    "reason",
+  ]);
+  return Boolean(
+    frame &&
+      frame.kind === "task.machine.retire" &&
+      isHostOpId(frame.opId) &&
+      isTaskMachineRetireTaskId(frame.taskId) &&
+      isTaskMachineRetireRunEpoch(frame.runEpoch) &&
+      (TASK_MACHINE_RETIRE_REASONS as readonly unknown[]).includes(
+        frame.reason,
+      ),
+  );
+}
+
+export function isTaskMachineRetireAckFrame(
+  value: unknown,
+): value is TaskMachineRetireAckFrame {
+  const frame = exactWireObject(
+    value,
+    ["kind", "opId", "taskId", "runEpoch", "outcome"],
+    ["error"],
+  );
+  return Boolean(
+    frame &&
+      frame.kind === "task.machine.retire.ack" &&
+      isHostOpId(frame.opId) &&
+      isTaskMachineRetireTaskId(frame.taskId) &&
+      isTaskMachineRetireRunEpoch(frame.runEpoch) &&
+      (TASK_MACHINE_RETIRE_OUTCOMES as readonly unknown[]).includes(
+        frame.outcome,
+      ) &&
+      (frame.error === undefined ||
+        (typeof frame.error === "string" &&
+          utf8ByteLength(frame.error) <=
+            MAX_TASK_MACHINE_RETIRE_ERROR_BYTES)),
+  );
+}
+
 export function isHostLogsStartFrame(
   value: unknown,
 ): value is HostLogsStartFrame {
@@ -4470,6 +4549,31 @@ export type GitHubRepositoryAccessErrorCode =
   | "github_unavailable"
   | "invalid_response";
 
+/**
+ * A one-task sidecar cannot prove a Files-v1 boundary closed in this boot, so
+ * it asks the cloud to stop the exact EC2 incarnation. `runEpoch` fences a
+ * delayed request from a successor assignment; the authenticated connection
+ * supplies the host identity and is deliberately not sender-controlled here.
+ */
+export interface TaskMachineRetireFrame {
+  kind: "task.machine.retire";
+  opId: string;
+  taskId: string;
+  runEpoch: number;
+  reason: TaskMachineRetireReason;
+}
+
+/** Durable acceptance, not completion: after `accepted` the cloud owns
+ * retrying the persisted exact-instance stop even if this socket disappears. */
+export interface TaskMachineRetireAckFrame {
+  kind: "task.machine.retire.ack";
+  opId: string;
+  taskId: string;
+  runEpoch: number;
+  outcome: TaskMachineRetireOutcome;
+  error?: string;
+}
+
 export type CloudToHost =
   | {
       kind: "command";
@@ -4633,6 +4737,7 @@ export type CloudToHost =
   | CredentialChangedFrame
   | CredentialRefreshResultFrame
   | CredentialsActivateResultFrame
+  | TaskMachineRetireAckFrame
   | McpCaptureProbeFrame
   | McpCaptureCompleteFrame
   | McpCaptureCancelFrame
@@ -4751,6 +4856,7 @@ export type HostToCloud =
   | CredentialRefreshFrame
   | CredentialsActivateFrame
   | CredentialsActivatedFrame
+  | TaskMachineRetireFrame
   | McpCaptureAckFrame
   | PersonCaptureAckFrame
   | GithubRevokeAckFrame