@runuai/host 0.9.149 → 0.9.151

package/lib/codex-auth.ts

@@ -59,6 +59,7 @@ import { dirname, join } from "node:path";
 
 import { isNull } from "drizzle-orm";
 
+import { onOwnCredentials, whileOnOwnCredentials } from "./cloud-credential-host";
 import { getDb, schema } from "./db";
 import { dockerCli } from "./docker-exec";
 import { brokeredFilesFor } from "./token-broker";
@@ -237,7 +238,9 @@ export async function injectCodexIntoContainer(
   const exists = deps.fileExists ?? existsSync;
   if (!exists(join(ownerCodexDir(), "auth.json"))) return true; // nothing to inject
   try {
-    await copyCodexInto(container, deps);
+    // ADR-154: no default slot once the runner begins leaving its own
+    // credentials; leaving waits for a copy under way.
+    await whileOnOwnCredentials(() => copyCodexInto(container, deps));
     return true;
   } catch (err) {
     console.error(
@@ -273,6 +276,9 @@ export async function injectCodexIntoMachine(
   },
   deps: CodexDeps = {},
 ): Promise<boolean> {
+  // ADR-154: no default slot once the runner begins leaving its own
+  // credentials (a start on them holds its permit through this).
+  if (!onOwnCredentials()) return true;
   const exists = deps.fileExists ?? existsSync;
   const dir = ownerCodexDir();
   if (!exists(join(dir, "auth.json"))) return true; // nothing to inject
@@ -317,6 +323,13 @@ export async function injectCodexIntoMachine(
  * failures are surfaced and don't stop the sweep.
  */
 export async function reinjectCodexRunningTasks(deps: CodexDeps = {}): Promise<void> {
+  // ADR-154: no default slot once the runner begins leaving its own
+  // credentials. Leaving waits for a sweep under way, and no copy starts
+  // after it began.
+  await whileOnOwnCredentials(() => reinjectOwnCodex(deps));
+}
+
+async function reinjectOwnCodex(deps: CodexDeps): Promise<void> {
   const exists = deps.fileExists ?? existsSync;
   if (!exists(join(ownerCodexDir(), "auth.json"))) return;
   const active = (deps.activeTaskContainers ?? defaultActiveTaskContainers)();
@@ -338,6 +351,7 @@ export async function reinjectCodexRunningTasks(deps: CodexDeps = {}): Promise<v
   if (targets.length === 0) return;
   console.log(`[codex] refreshing auth in ${targets.length} running task(s)`);
   for (const container of targets) {
+    if (!onOwnCredentials()) return;
     try {
       await copyCodexInto(container, deps, LIVE_CODEX_ITEMS);
     } catch (err) {
@@ -394,6 +408,11 @@ export function stopWatchingCodexAuth(): void {
  * Same sweep shape as the Codex reinject; only auth.json travels.
  */
 export async function reinjectGrokRunningTasks(deps: CodexDeps = {}): Promise<void> {
+  // ADR-154: as for Codex, only while the runner is on its own credentials.
+  await whileOnOwnCredentials(() => reinjectOwnGrok(deps));
+}
+
+async function reinjectOwnGrok(deps: CodexDeps): Promise<void> {
   const exists = deps.fileExists ?? existsSync;
   const src = join(process.env.UAI_OWNER_HOME?.trim() || homedir(), ".grok", "auth.json");
   if (!exists(src)) return;
@@ -417,6 +436,7 @@ export async function reinjectGrokRunningTasks(deps: CodexDeps = {}): Promise<vo
   try {
     writeFileSync(tmp, brokered, { mode: 0o600 });
     for (const container of active.filter((c) => running.has(c))) {
+      if (!onOwnCredentials()) break;
       const steps: Array<[string[], string]> = [
         [["exec", "-u", "root", ...runtimeAuthorityDockerExecArgs(), container, "/bin/mkdir", "-p", "/home/node/.grok"], "mkdir"],
         [["cp", tmp, `${container}:/home/node/.grok/auth.json`], "docker cp auth.json (brokered)"],

package/lib/container-runtime.ts

@@ -466,6 +466,82 @@ export async function claimContainerRuntimeForTask(
   );
 }
 
+let recheckRequester: ((reason: string) => void) | null = null;
+
+/** Who rechecks the runtime when work outside its activation asks: the
+ *  operator's recheck (main), which fences new work, re-probes, and runs
+ *  activation again, image maintenance (whose wedge sensor recycles a wedged
+ *  Apple runtime) and every persisted task's recovery. */
+export function setContainerRuntimeRecheckRequester(requester: ((reason: string) => void) | null): void {
+  recheckRequester = requester;
+}
+
+/**
+ * The requester main installs (#p2jdax). A forced recheck is never started
+ * over an activation under way: that would only make it stale. So:
+ *
+ * - an ask while an activation (with its retries) is under way is kept, and
+ *   asked again once that activation is done, since it may come after its
+ *   maintenance looked for a wedge; asks kept meanwhile become one;
+ * - an ask while only a forced recheck is under way joins it: the
+ *   activation it begins runs its maintenance after this ask;
+ * - otherwise it starts the operator's forced recheck.
+ */
+export function containerRuntimeRecheckRequester(deps: {
+  activationUnderWay: () => boolean;
+  activationSettled: () => Promise<void>;
+  recheck: () => Promise<void>;
+  warn?: (line: string) => void;
+}): (reason: string) => void {
+  const warn = deps.warn ?? ((line: string) => console.warn(line));
+  let underWay: Promise<void> | null = null;
+  let kept: string | null = null;
+  const ask = (reason: string): void => {
+    if (deps.activationUnderWay()) {
+      if (kept !== null) {
+        warn(`[host-agent] a container runtime recheck is already kept for after the activation under way (${reason})`);
+        return;
+      }
+      kept = reason;
+      warn(`[host-agent] a container runtime activation is under way; rechecking once it is done (${reason})`);
+      void deps.activationSettled().then(() => {
+        const next = kept;
+        kept = null;
+        if (next !== null) ask(next);
+      });
+      return;
+    }
+    if (underWay) {
+      warn(`[host-agent] a container runtime recheck is under way (${reason})`);
+      return;
+    }
+    warn(`[host-agent] rechecking the container runtime: ${reason}`);
+    underWay = deps
+      .recheck()
+      .catch((error: unknown) => {
+        warn(
+          `[host-agent] the requested container runtime recheck failed: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      })
+      .finally(() => {
+        underWay = null;
+      });
+  };
+  return ask;
+}
+
+/**
+ * Ask the runtime's owner to recheck it (#p2jdax): what work outside
+ * activation does instead of restarting the runtime's services itself,
+ * which would stop every task's VM with nothing to recover them. Returns
+ * whether anyone was there to ask.
+ */
+export function requestContainerRuntimeRecheck(reason: string): boolean {
+  if (!recheckRequester) return false;
+  recheckRequester(reason);
+  return true;
+}
+
 /** Explicit operator recovery: invalidate even a stale ready verdict before
  * probing so a healthy ready-to-ready result still requires a full activation
  * and persisted-task recovery pass. */

package/lib/credential-cleanup-inventory.ts

@@ -0,0 +1,405 @@
+/**
+ * ADR-154's durable, non-secret inventory of task-local credential paths.
+ *
+ * Writers remember a path before placing credential material there. Cleanup
+ * later removes the row only after it has independently verified the path is
+ * absent. This module records the witness; it deliberately does not perform
+ * cleanup or inspect credential contents.
+ */
+import { and, asc, eq, lte, sql } from "drizzle-orm";
+
+import { idPathSegment } from "../src/cloud-credentials";
+import type {
+  HostCredentialCleanupPath,
+  HostCredentialCleanupPending,
+  HostLeasedEngineRuntime,
+} from "../db/schema";
+import { getDb, schema, type Db } from "./db";
+import { assertSafeHostTaskId } from "./task-identity";
+import {
+  LEASE_AWS_CONFIG_FILE,
+  LEASE_CREDENTIALS_FILE,
+} from "./lease";
+import {
+  serializeTaskEnvironmentLocator,
+  type TaskEnvironmentLocator,
+} from "./task-environment/types";
+
+type InventoryExecutor = Pick<Db, "select" | "insert" | "delete">;
+
+export interface CredentialCleanupPathKey {
+  taskId: string;
+  runEpoch: number;
+  runtime: TaskEnvironmentLocator;
+  lane: CredentialCleanupLane;
+  path: string;
+}
+
+export type CredentialCleanupLane =
+  | "engine-auth"
+  | "github"
+  | "ssh"
+  | "files-session";
+
+export type CredentialCleanupPendingReason =
+  | "removed"
+  | "denied"
+  | "expired"
+  | "assignment"
+  | "restart"
+  | "stop"
+  | "park"
+  | "teardown";
+
+const CLEANUP_PENDING_REASONS = new Set<CredentialCleanupPendingReason>([
+  "removed",
+  "denied",
+  "expired",
+  "assignment",
+  "restart",
+  "stop",
+  "park",
+  "teardown",
+]);
+
+export interface CredentialCleanupPendingInput {
+  taskId: string;
+  runEpoch: number;
+  reason: CredentialCleanupPendingReason;
+}
+
+export interface CredentialCleanupPathInput extends CredentialCleanupPathKey {
+  credentialId: string;
+}
+
+function positiveRunEpoch(value: number): void {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new Error("credential cleanup run epoch must be a positive safe integer");
+  }
+}
+
+function normalizedPending(input: CredentialCleanupPendingInput): void {
+  assertSafeHostTaskId(input.taskId);
+  positiveRunEpoch(input.runEpoch);
+  if (!CLEANUP_PENDING_REASONS.has(input.reason)) {
+    throw new Error("credential cleanup reason is invalid");
+  }
+}
+
+/**
+ * Durable fail-closed witness written before cleanup closes any gate or starts
+ * any asynchronous work.  A newer assignment can only raise the recorded
+ * epoch; a delayed older cleanup must never move the boot-recovery floor back.
+ */
+export function rememberCredentialCleanupPending(
+  input: CredentialCleanupPendingInput,
+  db: InventoryExecutor = getDb(),
+  now = Date.now(),
+): void {
+  normalizedPending(input);
+  db.insert(schema.hostCredentialCleanupPending)
+    .values({
+      taskId: input.taskId,
+      runEpoch: input.runEpoch,
+      reason: input.reason,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .onConflictDoUpdate({
+      target: schema.hostCredentialCleanupPending.taskId,
+      set: {
+        runEpoch: sql`max(${schema.hostCredentialCleanupPending.runEpoch}, excluded.run_epoch)`,
+        reason: sql`CASE WHEN excluded.run_epoch >= ${schema.hostCredentialCleanupPending.runEpoch} THEN excluded.reason ELSE ${schema.hostCredentialCleanupPending.reason} END`,
+        updatedAt: now,
+      },
+    })
+    .run();
+}
+
+export function listCredentialCleanupPending(
+  db: InventoryExecutor = getDb(),
+): HostCredentialCleanupPending[] {
+  return db
+    .select()
+    .from(schema.hostCredentialCleanupPending)
+    .orderBy(asc(schema.hostCredentialCleanupPending.taskId))
+    .all();
+}
+
+/** Clear only a marker no newer than the cleanup that just succeeded. */
+export function forgetCredentialCleanupPending(
+  taskId: string,
+  throughRunEpoch: number,
+  db: InventoryExecutor = getDb(),
+): void {
+  assertSafeHostTaskId(taskId);
+  positiveRunEpoch(throughRunEpoch);
+  db.delete(schema.hostCredentialCleanupPending)
+    .where(
+      and(
+        eq(schema.hostCredentialCleanupPending.taskId, taskId),
+        lte(schema.hostCredentialCleanupPending.runEpoch, throughRunEpoch),
+      ),
+    )
+    .run();
+}
+
+/** An engine account's file, or the intermediate its writer stages
+ * (`.uai-tmp`) before renaming it into place. */
+const ENGINE_AUTH_PATH = /^\/home\/node\/(?:\.codex(?:-acct-[A-Za-z0-9_-]+)?(?:-agent-[A-Za-z0-9_-]+)?|\.grok(?:-acct-[A-Za-z0-9_-]+)?|\.local\/share\/opencode(?:-acct-[A-Za-z0-9_-]+)?)\/auth\.json(?:\.uai-tmp)?$/;
+/** A file a writer stages (`.uai-tmp`) before renaming it into place. */
+function andStaged(...paths: string[]): string[] {
+  return paths.flatMap((path) => [path, `${path}.uai-tmp`]);
+}
+const GITHUB_PATHS = new Set(
+  andStaged("/home/node/.config/gh/hosts.yml", "/home/node/.config/gh/uai.gitconfig"),
+);
+const SSH_PATHS = new Set(
+  andStaged(
+    "/home/node/.ssh/id_ed25519",
+    "/home/node/.ssh/id_ed25519.pub",
+    "/home/node/.ssh/allowed_signers",
+    "/home/node/.ssh/uai-signing.gitconfig",
+    "/home/node/.ssh/uai-transport.gitconfig",
+  ),
+);
+const FILES_SESSION_PATHS = new Set([
+  LEASE_CREDENTIALS_FILE,
+  LEASE_AWS_CONFIG_FILE,
+]);
+
+/** Destructive cleanup accepts only lane-owned files. The executor repeats
+ * this check after reconstructing the runtime and resolves every parent
+ * symlink-safely before deletion; persistence never turns an arbitrary task
+ * path into cleanup authority. */
+export function assertCredentialCleanupPath(
+  lane: CredentialCleanupLane,
+  value: string,
+): void {
+  if (
+    typeof value !== "string" ||
+    value.length === 0 ||
+    value.length > 4096 ||
+    value.includes("\0") ||
+    value.split("/").some((part) => part === "." || part === "..")
+  ) {
+    throw new Error("credential cleanup path is invalid");
+  }
+  const allowed =
+    (lane === "engine-auth" && ENGINE_AUTH_PATH.test(value)) ||
+    (lane === "github" && GITHUB_PATHS.has(value)) ||
+    (lane === "ssh" && SSH_PATHS.has(value)) ||
+    (lane === "files-session" && FILES_SESSION_PATHS.has(value));
+  if (!allowed) {
+    throw new Error("credential cleanup path is outside its managed lane");
+  }
+}
+
+function normalizedKey(input: CredentialCleanupPathKey): {
+  taskId: string;
+  runEpoch: number;
+  runtimeLocator: string;
+  lane: CredentialCleanupLane;
+  path: string;
+} {
+  assertSafeHostTaskId(input.taskId);
+  positiveRunEpoch(input.runEpoch);
+  assertCredentialCleanupPath(input.lane, input.path);
+  return {
+    taskId: input.taskId,
+    runEpoch: input.runEpoch,
+    runtimeLocator: serializeTaskEnvironmentLocator(input.runtime),
+    lane: input.lane,
+    path: input.path,
+  };
+}
+
+/** Record before writing the credential path. Re-recording the same exact
+ * runtime path is idempotent and preserves its first-seen timestamp. */
+export function rememberCredentialCleanupPath(
+  input: CredentialCleanupPathInput,
+  db: InventoryExecutor = getDb(),
+  now = Date.now(),
+): void {
+  // The shared path encoder has the contract's exact bounded id grammar.
+  idPathSegment(input.credentialId);
+  const key = normalizedKey(input);
+  db.insert(schema.hostCredentialCleanupPaths)
+    .values({
+      ...key,
+      credentialId: input.credentialId,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .onConflictDoUpdate({
+      target: [
+        schema.hostCredentialCleanupPaths.taskId,
+        schema.hostCredentialCleanupPaths.runEpoch,
+        schema.hostCredentialCleanupPaths.runtimeLocator,
+        schema.hostCredentialCleanupPaths.path,
+      ],
+      set: { credentialId: input.credentialId, updatedAt: now },
+    })
+    .run();
+}
+
+/** All outstanding cleanup work for a task, oldest assignment first. */
+export function listCredentialCleanupPaths(
+  taskId: string,
+  db: InventoryExecutor = getDb(),
+): HostCredentialCleanupPath[] {
+  assertSafeHostTaskId(taskId);
+  return db
+    .select()
+    .from(schema.hostCredentialCleanupPaths)
+    .where(eq(schema.hostCredentialCleanupPaths.taskId, taskId))
+    .orderBy(
+      asc(schema.hostCredentialCleanupPaths.runEpoch),
+      asc(schema.hostCredentialCleanupPaths.runtimeLocator),
+      asc(schema.hostCredentialCleanupPaths.path),
+    )
+    .all();
+}
+
+/** Boot recovery must not depend on a current manifest or even a surviving
+ * host-task row. Return the entire non-secret worklist in deterministic order. */
+export function listAllCredentialCleanupPaths(
+  db: InventoryExecutor = getDb(),
+): HostCredentialCleanupPath[] {
+  return db
+    .select()
+    .from(schema.hostCredentialCleanupPaths)
+    .orderBy(
+      asc(schema.hostCredentialCleanupPaths.taskId),
+      asc(schema.hostCredentialCleanupPaths.runEpoch),
+      asc(schema.hostCredentialCleanupPaths.runtimeLocator),
+      asc(schema.hostCredentialCleanupPaths.path),
+    )
+    .all();
+}
+
+/**
+ * Value-free boot worklist. A marker keeps a task recoverable even when a
+ * crash happened before its first path was inventoried, or after every path
+ * was deleted but before in-memory material was dropped.
+ */
+export function listCredentialCleanupRecoveryTasks(
+  db: InventoryExecutor = getDb(),
+): Array<{ taskId: string; runEpoch: number }> {
+  const tasks = new Map<string, number>();
+  for (const pending of listCredentialCleanupPending(db)) {
+    tasks.set(pending.taskId, pending.runEpoch);
+  }
+  for (const row of listAllCredentialCleanupPaths(db)) {
+    tasks.set(row.taskId, Math.max(tasks.get(row.taskId) ?? 0, row.runEpoch));
+  }
+  // A leased engine on an env-token account leaves no path: the record of
+  // where it ran is the evidence its task needs a cleanup after a crash.
+  for (const row of listAllLeasedEngineRuntimes(db)) {
+    tasks.set(row.taskId, Math.max(tasks.get(row.taskId) ?? 0, row.runEpoch));
+  }
+  return [...tasks]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([taskId, runEpoch]) => ({ taskId, runEpoch }));
+}
+
+/** Persistence primitive for the later cleanup executor. The executor is
+ * responsible for proving absence before it calls this. */
+export function forgetCredentialCleanupPath(
+  input: CredentialCleanupPathKey,
+  db: InventoryExecutor = getDb(),
+): void {
+  const key = normalizedKey(input);
+  db.delete(schema.hostCredentialCleanupPaths)
+    .where(
+      and(
+        eq(schema.hostCredentialCleanupPaths.taskId, key.taskId),
+        eq(schema.hostCredentialCleanupPaths.runEpoch, key.runEpoch),
+        eq(
+          schema.hostCredentialCleanupPaths.runtimeLocator,
+          key.runtimeLocator,
+        ),
+        eq(schema.hostCredentialCleanupPaths.lane, key.lane),
+        eq(schema.hostCredentialCleanupPaths.path, key.path),
+      ),
+    )
+    .run();
+}
+
+/**
+ * ADR-154 §6: record, before a leased engine starts in it, the runtime it
+ * starts in for this run, so the cleanup proves its engines stopped there
+ * even when they left no file to inventory. Idempotent.
+ */
+export function rememberLeasedEngineRuntime(
+  input: { taskId: string; runEpoch: number; runtime: TaskEnvironmentLocator },
+  db: InventoryExecutor = getDb(),
+  now = Date.now(),
+): void {
+  assertSafeHostTaskId(input.taskId);
+  positiveRunEpoch(input.runEpoch);
+  db.insert(schema.hostLeasedEngineRuntimes)
+    .values({
+      taskId: input.taskId,
+      runEpoch: input.runEpoch,
+      runtimeLocator: serializeTaskEnvironmentLocator(input.runtime),
+      createdAt: now,
+    })
+    .onConflictDoNothing()
+    .run();
+}
+
+/** Every runtime a leased engine of the task was started in, oldest run first. */
+export function listLeasedEngineRuntimes(
+  taskId: string,
+  db: InventoryExecutor = getDb(),
+): HostLeasedEngineRuntime[] {
+  assertSafeHostTaskId(taskId);
+  return db
+    .select()
+    .from(schema.hostLeasedEngineRuntimes)
+    .where(eq(schema.hostLeasedEngineRuntimes.taskId, taskId))
+    .orderBy(
+      asc(schema.hostLeasedEngineRuntimes.runEpoch),
+      asc(schema.hostLeasedEngineRuntimes.runtimeLocator),
+    )
+    .all();
+}
+
+/** Every runtime a leased engine of any task was started in. */
+export function listAllLeasedEngineRuntimes(
+  db: InventoryExecutor = getDb(),
+): HostLeasedEngineRuntime[] {
+  return db
+    .select()
+    .from(schema.hostLeasedEngineRuntimes)
+    .orderBy(
+      asc(schema.hostLeasedEngineRuntimes.taskId),
+      asc(schema.hostLeasedEngineRuntimes.runEpoch),
+      asc(schema.hostLeasedEngineRuntimes.runtimeLocator),
+    )
+    .all();
+}
+
+/** After a whole-task cleanup proved every engine of these runs stopped in
+ *  these runtimes: forget them, never a newer run's. */
+export function forgetLeasedEngineRuntimes(
+  taskId: string,
+  throughRunEpoch: number,
+  runtimeLocators: readonly string[],
+  db: InventoryExecutor = getDb(),
+): void {
+  assertSafeHostTaskId(taskId);
+  positiveRunEpoch(throughRunEpoch);
+  for (const runtimeLocator of runtimeLocators) {
+    db.delete(schema.hostLeasedEngineRuntimes)
+      .where(
+        and(
+          eq(schema.hostLeasedEngineRuntimes.taskId, taskId),
+          lte(schema.hostLeasedEngineRuntimes.runEpoch, throughRunEpoch),
+          eq(schema.hostLeasedEngineRuntimes.runtimeLocator, runtimeLocator),
+        ),
+      )
+      .run();
+  }
+}