@runuai/host 0.9.148 → 0.9.150

package/lib/secrets-proxy/task-trust.ts

@@ -0,0 +1,574 @@
+/**
+ * Putting a task behind the proxy (ADR-153 §3 "Order") and keeping it there:
+ * what task-up.sh runs inside the container before uai-init, and the same
+ * steps for a running task. Every root mutation of a task (its CA, its pins)
+ * goes through one queue per task, so no two ever interleave and the
+ * recorded readiness is always that of the last one to land; shutdown stops
+ * the queues and waits for them.
+ */
+
+import { chmodSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { isSafeHostTaskId } from "../task-identity";
+import { SELF_TASK_ENVIRONMENT_PROVIDER, SELF_TASK_USER, TASK_HOSTS_VIEW_PATH } from "../task-environment/self-identity";
+import type { TaskEnvironmentHandle } from "../task-environment/types";
+import { MCP_GATEWAY_PORT } from "../task-gateway-port";
+import { taskViewProven } from "./sidecar-view";
+import { installTaskTrust, reconcileTaskPins, type TaskNetAdmin } from "./container-steps";
+import {
+  beginTaskStart,
+  endTaskStart,
+  forgetTaskProxyReadiness,
+  isTaskStarting,
+  recordTaskProxyReadiness,
+  taskProxyReadiness,
+  type ProxyStepOutcome,
+} from "./readiness";
+import type { SecretsProxyRuntime } from "./runtime";
+import { caInstallScript, hostsReconcileScript, redirectChainScript } from "./task-scripts";
+
+/** Where the task forwarder listens (both launchers pass `--listen-port
+ *  5877`): the NAT chain sends a pinned name's :443 here. */
+export const TASK_FORWARDER_LISTEN_PORT = 5877;
+
+/** How the proxy's steps run on one runtime. */
+interface ProxyRuntime {
+  /** Where the redirect step's NET_ADMIN comes from. */
+  readonly netAdmin: TaskNetAdmin;
+  /** Where the chain sends a pinned name's :443. */
+  readonly listenPort: () => number;
+  /** Whose traffic it redirects, where not everyone's in the namespace. */
+  readonly uidOwner?: string;
+  /** Where the names are pinned, where /etc/hosts is not the task's alone,
+   *  and whether every process of the task sees them. */
+  readonly hostsView?: {
+    readonly path: string;
+    readonly from: string;
+    readonly proven: (taskId: string) => boolean;
+  };
+}
+
+/**
+ * The runtimes this runner puts behind the proxy (the CA and the pins):
+ * - Docker: the redirect gets NET_ADMIN for that one exec (`--privileged`),
+ *   to the task's forwarder.
+ * - Apple: the task VM holds NET_ADMIN from its creation (`--cap-add
+ *   CAP_NET_ADMIN`), so a root exec has it (the provider refuses
+ *   `privileged`); to the task's forwarder.
+ * - A Uai Cloud machine's own task (`self`): the sidecar is root on the
+ *   machine and runs the steps itself. The chain sends only the task user's
+ *   traffic, to the sidecar's own gateway, where a loopback TLS connection
+ *   is its one task's. The names are pinned only in the task's own view of
+ *   the hosts, which every task process gets bind-mounted over /etc/hosts:
+ *   the machine's /etc/hosts, and so every root process the sidecar runs
+ *   (its SSM refresh, its S3 mounts, its own updates), never sees them.
+ * A machine an owned runner drives over ssh is not among them: its proxied
+ * keys stay withheld.
+ */
+const RUNTIMES: ReadonlyMap<string, ProxyRuntime> = new Map<string, ProxyRuntime>([
+  ["docker-compose", { netAdmin: "per-exec", listenPort: () => TASK_FORWARDER_LISTEN_PORT }],
+  ["apple-container", { netAdmin: "from-creation", listenPort: () => TASK_FORWARDER_LISTEN_PORT }],
+  [
+    SELF_TASK_ENVIRONMENT_PROVIDER,
+    {
+      netAdmin: "root",
+      listenPort: () => MCP_GATEWAY_PORT,
+      uidOwner: SELF_TASK_USER,
+      hostsView: { path: TASK_HOSTS_VIEW_PATH, from: "/etc/hosts", proven: taskViewProven },
+    },
+  ],
+]);
+
+/** Whether this runner puts tasks of `provider` behind the proxy. */
+export function isProxiedRuntime(provider: string | null | undefined): boolean {
+  return provider !== null && provider !== undefined && RUNTIMES.has(provider);
+}
+
+/** A task as the runner's root steps use it: its identity and `exec`. */
+export type TaskRootExec = Pick<TaskEnvironmentHandle<unknown>, "exec" | "descriptor">;
+type Exec = TaskRootExec;
+
+function takesTrust(environment: Exec): boolean {
+  return isProxiedRuntime(environment.descriptor.locator.provider);
+}
+
+// --- one mutation queue per task -------------------------------------------
+
+const queues = new Map<string, Promise<unknown>>();
+let quiescing = false;
+
+/** Bumped at a stop, a deletion and a new start: every write queued or under
+ *  way before is void from then on (it runs no further exec and records
+ *  nothing), so none can land over, or recreate readiness for, what follows. */
+const epochs = new Map<string, number>();
+const epochOf = (taskId: string): number => epochs.get(taskId) ?? 0;
+
+export function invalidateTaskWrites(taskId: string): void {
+  epochs.set(taskId, epochOf(taskId) + 1);
+}
+
+/** Whether the runner is shutting down: nothing root may start in a task. */
+export function taskWritesQuiescing(): boolean {
+  return quiescing;
+}
+
+/** True while the operation may still act: the runner is not shutting down,
+ *  no stop, deletion or start has voided it, and its caller's own check (a
+ *  recovery's generation) still holds. */
+type Live = () => boolean;
+
+/** Why an operation that is no longer live stops. */
+function refusal(): Error {
+  return new Error(quiescing ? "the runner is shutting down" : "this write was superseded");
+}
+
+/** Runs `step` after every earlier write of the task has ended. Refused once
+ *  shutdown has begun: a write that started after its runner's last moment
+ *  could land over the replacement's. */
+function mutate<T>(taskId: string, step: (live: Live) => Promise<T>, isCurrent?: () => boolean): Promise<T> {
+  if (quiescing) return Promise.reject(new Error("the runner is shutting down"));
+  const epoch = epochOf(taskId);
+  const live: Live = () => !quiescing && epochOf(taskId) === epoch && (isCurrent === undefined || isCurrent());
+  const previous = queues.get(taskId) ?? Promise.resolve();
+  const next = previous
+    .catch(() => undefined)
+    .then(() => {
+      if (!live()) throw refusal();
+      return step(live);
+    });
+  queues.set(taskId, next);
+  const settled = (): void => {
+    if (queues.get(taskId) === next) queues.delete(taskId);
+  };
+  next.then(settled, settled);
+  return next;
+}
+
+/** Every exec of a queued step goes through this: once the operation is not
+ *  live (shutdown, a stop, a deletion, a new start, a stale recovery), no
+ *  further exec starts, even inside an operation already under way, and
+ *  not even one already asked for whose provider is still looking up where
+ *  to run it (the request's `admit`). The
+ *  steps are ordered so stopping between any two leaves nothing unsafe (the
+ *  chain before the names when pinning, the names first when unpinning), and
+ *  whatever writes next reconciles what is left. */
+function guarded(environment: Exec, live: Live): Exec {
+  return {
+    descriptor: environment.descriptor,
+    exec: (request) => {
+      if (!live()) return Promise.reject(refusal());
+      // And again where the provider starts the process: a machine looks
+      // up its address first, which can take as long as a cloud API call.
+      const admit = (): void => {
+        request.admit?.();
+        if (!live()) throw refusal();
+      };
+      return environment.exec({ ...request, admit });
+    },
+  };
+}
+
+/** Shutdown: refuses every new write and exec, and resolves once those under
+ *  way (a recovery's CA install, a re-pin, a forwarder launch) have ended. */
+export async function quiesceTaskProxyMutations(): Promise<void> {
+  quiescing = true;
+  await Promise.all([...queues.values()].map((queued) => queued.catch(() => undefined)));
+}
+
+/**
+ * Any other root step a lifecycle takes in a task (the recovery's forwarder
+ * launch), run in the same queue and under the same fence: never once
+ * shutdown has begun, a stop, deletion or start has voided it, or
+ * `isCurrent` says the operation is stale; joined by shutdown. The step
+ * reaches the task only through the `environment` it is given, whose every
+ * exec is fenced when it is made, not only when the step begins: the
+ * launch awaits the gateway's address, and on a machine installs its
+ * assets, before the exec that starts the forwarder.
+ */
+export function runTaskRootStep<T>(
+  environment: TaskRootExec,
+  step: (environment: TaskRootExec, live: () => boolean) => Promise<T>,
+  isCurrent?: () => boolean,
+): Promise<T> {
+  return mutate(environment.descriptor.taskId, (live) => step(guarded(environment, live), live), isCurrent);
+}
+
+/**
+ * A start's exclusive hold on the task's writes, from before its root steps
+ * until its result is recorded. Earlier writers are voided (they finish at
+ * most the exec under way) and waited for; none runs until the returned
+ * release. Rejects once shutdown has begun.
+ */
+function holdTaskWritesForStart(taskId: string): Promise<() => void> {
+  invalidateTaskWrites(taskId);
+  return new Promise((resolve, reject) => {
+    mutate(
+      taskId,
+      () =>
+        new Promise<void>((release) => {
+          resolve(release);
+        }),
+    ).catch(reject);
+  });
+}
+
+/** The holds of the starts under way, each ended once (finish or close). */
+const startHolds = new Map<string, () => void>();
+
+function endStartHold(taskId: string): void {
+  const release = startHolds.get(taskId);
+  startHolds.delete(taskId);
+  release?.();
+}
+
+// --- the steps, run only inside a task's queue --------------------------------
+
+async function installStep(environment: Exec, live: Live, bundle: string): Promise<void> {
+  const { taskId } = environment.descriptor;
+  try {
+    await installTaskTrust(environment, bundle);
+  } catch (error) {
+    if (live()) recordTaskProxyReadiness(taskId, { trust: "failed", bundle: null });
+    throw error;
+  }
+  if (live()) recordTaskProxyReadiness(taskId, { trust: "installed", bundle });
+}
+
+/** Unpinned tasks, while the proxy is off: their pins are known to be gone. */
+const unpinned = new Set<string>();
+
+/** With the proxy off: the pins out, once (recorded until the task pins
+ *  again); a failure is not recorded, so the next sweep tries again. */
+async function unpinStep(environment: Exec, live: Live): Promise<void> {
+  const { taskId } = environment.descriptor;
+  if (unpinned.has(taskId)) return;
+  await pinStep(environment, live, []);
+  if (live()) {
+    forgetTaskProxyReadiness(taskId);
+    unpinned.add(taskId);
+  }
+}
+
+async function pinStep(environment: Exec, live: Live, domains: readonly string[]): Promise<void> {
+  const { taskId } = environment.descriptor;
+  try {
+    const proxied = RUNTIMES.get(environment.descriptor.locator.provider);
+    if (proxied === undefined) throw new Error("the proxy does not run on this task's runtime");
+    await reconcileTaskPins(environment, {
+      domains,
+      listenPort: proxied.listenPort(),
+      netAdmin: proxied.netAdmin,
+      ...(proxied.uidOwner === undefined ? {} : { uidOwner: proxied.uidOwner }),
+      ...(proxied.hostsView === undefined
+        ? {}
+        : {
+            hostsView: {
+              path: proxied.hostsView.path,
+              from: proxied.hostsView.from,
+              proven: () => proxied.hostsView?.proven(taskId) === true,
+            },
+          }),
+    });
+  } catch (error) {
+    // Whatever was pinned may now be half-rewritten: nothing counts as pinned.
+    if (live()) recordTaskProxyReadiness(taskId, { pins: "failed", pinnedDomains: [] });
+    unpinned.delete(taskId);
+    throw error;
+  }
+  if (live()) {
+    recordTaskProxyReadiness(taskId, { pins: domains.length > 0 ? "installed" : "skipped", pinnedDomains: [...domains] });
+  }
+}
+
+function sameDomains(left: readonly string[], right: readonly string[]): boolean {
+  return left.length === right.length && left.every((domain, index) => domain === right[index]);
+}
+
+// --- task-up ---------------------------------------------------------------
+
+function stepsDir(dataDir: string, taskId: string): string {
+  if (!isSafeHostTaskId(taskId)) throw new Error("invalid host task identity");
+  return join(dataDir, "secrets-proxy", "task-up", taskId);
+}
+
+/**
+ * Begins a task-up behind the proxy. First the start takes the task's writes
+ * (holdTaskWritesForStart), so no older writer (a recovery, a re-pin) lands
+ * over its steps and task-up is the task's only writer until its result is
+ * recorded. Then it writes what task-up.sh runs as root inside the
+ * container: the CA bundle, its install script, and the chain and pin
+ * scripts for the task's current domains, with the order to run them in.
+ * Public material only (certificates and fixed scripts). task-up writes each
+ * step's outcome back into the same directory as it goes.
+ *
+ * Returns the directory, passed to task-up.sh as UAI_SECRETS_PROXY_DIR, or
+ * null once the runner is shutting down: the task then starts without the
+ * proxy's steps, its proxied keys withheld, and the next runner reconciles
+ * it. finishTaskUpTrustSteps ends the hold; closeTaskUpTrustSteps ends it
+ * for a start that never got that far.
+ */
+export async function writeTaskUpTrustSteps(
+  dataDir: string,
+  taskId: string,
+  runtime: SecretsProxyRuntime,
+): Promise<string | null> {
+  const dir = stepsDir(dataDir, taskId);
+  let release: () => void;
+  try {
+    release = await holdTaskWritesForStart(taskId);
+  } catch {
+    forgetTaskProxyReadiness(taskId);
+    return null;
+  }
+  startHolds.set(taskId, release);
+  try {
+    rmSync(dir, { recursive: true, force: true });
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    chmodSync(dir, 0o700);
+    const bundle = runtime.caBundlePem();
+    const domains = runtime.domainsFor(taskId);
+    const files: Record<string, string> = {
+      "ca-bundle.pem": bundle,
+      "ca-install.sh": caInstallScript(),
+      "pins.sh": hostsReconcileScript(domains),
+      "chain.sh": redirectChainScript({ toPort: TASK_FORWARDER_LISTEN_PORT, enabled: domains.length > 0 }),
+      // As for a running task: the chain before the names when pinning, the
+      // names first when there is nothing to pin.
+      "routing-order": domains.length > 0 ? "pin\n" : "unpin\n",
+    };
+    for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body, { mode: 0o600 });
+    unpinned.delete(taskId);
+    beginTaskStart(taskId, { dir, bundle, domains });
+  } catch (error) {
+    forgetTaskProxyReadiness(taskId);
+    endStartHold(taskId);
+    throw error;
+  }
+  return dir;
+}
+
+/** Ends a task-up: what it reported becomes the task's readiness (nothing,
+ *  when it failed), its directory goes, and its hold on the task's writes
+ *  ends, so a writer queued meanwhile reconciles from what it recorded. */
+export function finishTaskUpTrustSteps(
+  dataDir: string,
+  taskId: string,
+  result: { secretsProxyTrust?: ProxyStepOutcome; secretsProxyPins?: ProxyStepOutcome } | null,
+): void {
+  try {
+    const pending = endTaskStart(taskId);
+    if (isSafeHostTaskId(taskId)) rmSync(stepsDir(dataDir, taskId), { recursive: true, force: true });
+    if (pending === undefined || result === null) {
+      forgetTaskProxyReadiness(taskId);
+      return;
+    }
+    const trust = result.secretsProxyTrust ?? "skipped";
+    const pins = result.secretsProxyPins ?? "skipped";
+    recordTaskProxyReadiness(taskId, {
+      trust,
+      bundle: trust === "installed" ? pending.bundle : null,
+      pins,
+      pinnedDomains: pins === "installed" ? pending.domains : [],
+      // task-up fails outright unless its forwarder started.
+      forwarder: true,
+    });
+  } finally {
+    endStartHold(taskId);
+  }
+}
+
+/** The end of every start, whatever happened: one that never reported
+ *  counts as failed (and its hold ends). Nothing once it was finished. */
+export function closeTaskUpTrustSteps(dataDir: string, taskId: string): void {
+  if (isTaskStarting(taskId)) finishTaskUpTrustSteps(dataDir, taskId, null);
+}
+
+// --- reconciling a running task -----------------------------------------------
+
+/**
+ * Brings one running Docker or Apple task to where it should be, doing only
+ * what differs from its recorded readiness, in its queue:
+ * - proxy on: the current CA bundle, then pins for its current domains;
+ * - proxy off (`runtime` null): no pins, so formerly proxied names go
+ *   direct again.
+ * Any other runtime is left untouched. Level-triggered, so every path that
+ * may have missed something (the end of a start, a change of policy, the
+ * trust watcher's retries, the proxy-off sweep) simply calls it again.
+ * Throws if a step failed; what did land is recorded.
+ */
+export function reconcileTaskProxyState(
+  target: Exec,
+  runtime: SecretsProxyRuntime | null,
+  isCurrent?: () => boolean,
+): Promise<void> {
+  if (!takesTrust(target)) return Promise.resolve();
+  const { taskId } = target.descriptor;
+  return mutate(
+    taskId,
+    async (live) => {
+      const environment = guarded(target, live);
+      if (runtime === null) {
+        await unpinStep(environment, live);
+        if (!live()) throw refusal();
+        return;
+      }
+      unpinned.delete(taskId);
+      const bundle = runtime.caBundlePem();
+      const ready = taskProxyReadiness(taskId);
+      if (ready.trust !== "installed" || ready.bundle !== bundle) await installStep(environment, live, bundle);
+      // Voided while its CA exec ran (a stop, a recovery beginning): it ends
+      // there, and what it read before counts for nothing.
+      if (!live()) throw refusal();
+      const domains = runtime.domainsFor(taskId);
+      const after = taskProxyReadiness(taskId);
+      const pinned = domains.length > 0 ? after.pins === "installed" : after.pins === "skipped";
+      if (!pinned || !sameDomains(after.pinnedDomains, domains)) {
+        // Never a name pinned with nothing listening behind it: until a
+        // start or a recovery has the task's forwarder running, the pins
+        // wait for it (a recovery pins in the same slot as its launch).
+        // Taking pins out needs no forwarder.
+        if (domains.length > 0 && !after.forwarder) {
+          throw new Error("the task's forwarder is not known to run; its next recovery pins it");
+        }
+        await pinStep(environment, live, domains);
+        // Its last exec may have been the one under way when it was voided:
+        // landed, but recorded nowhere, so it does not count as done.
+        if (!live()) throw refusal();
+      }
+    },
+    isCurrent,
+  );
+}
+
+/** A proxy step that failed during a recovery: the task comes up without
+ *  the proxy, its proxied keys withheld, and the trust watcher retries. */
+export interface RecoveryProxyFailure {
+  readonly error: unknown;
+}
+
+/**
+ * A task's root steps in the order a start takes them (ADR-153 "Order"), in
+ * one slot of its queue so no other writer lands between them: for a
+ * recovery, and for a machine sidecar's start and resume, which run the
+ * steps themselves rather than through task-up.sh. Each step runs again
+ * whatever is recorded: a restarted runner records nothing, and a restarted
+ * VM or machine keeps none of it.
+ * - Proxy on: the CA, then `launch` (the task's forwarder; nothing on a
+ *   machine, whose sidecar serves the task from its own gateway), then the
+ *   pins for its current domains.
+ * - Proxy off: the pins out, then `launch`.
+ * No name is pinned before its forwarder runs. A launch that fails pins
+ * nothing, takes out the pins the task still holds, and rejects, and the
+ * caller keeps consumers out. A failed proxy
+ * step does not stop the launch; it is returned, and after a failed CA
+ * nothing is pinned either. `proxy` null (a baked task), or a runtime the
+ * runner does not put behind the proxy, runs `launch` alone. A stop,
+ * deletion, start, stale recovery or shutdown rejects the slot at whichever
+ * step it meets.
+ */
+export function applyTaskRootSteps(
+  target: Exec,
+  proxy: { readonly runtime: SecretsProxyRuntime | null } | null,
+  launch: (environment: TaskRootExec) => Promise<void>,
+  isCurrent?: () => boolean,
+): Promise<RecoveryProxyFailure | null> {
+  const { taskId } = target.descriptor;
+  const runtime = proxy?.runtime ?? null;
+  const proxied = proxy !== null && takesTrust(target);
+  return runTaskRootStep(
+    target,
+    async (environment, live) => {
+      let failure: RecoveryProxyFailure | null = null;
+      // A step's failure is the proxy's only while the slot is live; one
+      // that met a stop, a stale recovery or shutdown ends the slot.
+      const attempt = async (step: () => Promise<void>): Promise<void> => {
+        try {
+          await step();
+        } catch (error) {
+          if (!live()) throw error;
+          failure = { error };
+          return;
+        }
+        // Its exec may have been the one under way when the slot stopped.
+        if (!live()) throw refusal();
+      };
+      if (proxied) {
+        // Replayed because a restart may have taken all of it: nothing
+        // recorded before counts until a step here lands it again, so a step
+        // that fails leaves the task undeliverable, never ready on the
+        // strength of a VM that is gone.
+        forgetTaskProxyReadiness(taskId);
+        if (runtime === null) {
+          await attempt(() => unpinStep(environment, live));
+        } else {
+          unpinned.delete(taskId);
+          await attempt(() => installStep(environment, live, runtime.caBundlePem()));
+        }
+      }
+      if (!live()) throw refusal();
+      try {
+        await launch(environment);
+      } catch (error) {
+        // No forwarder, so no name may point at one. Pins already in the task
+        // (the earlier VM's, or the last exec of a reconcile voided while it
+        // ran, which no fence can call back) come out, as far as the task
+        // lets them.
+        if (proxied && runtime !== null && live()) await pinStep(environment, live, []).catch(() => undefined);
+        throw error;
+      }
+      if (!live()) throw refusal();
+      if (proxied) recordTaskProxyReadiness(taskId, { forwarder: true });
+      if (proxied && runtime !== null && failure === null) {
+        await attempt(() => pinStep(environment, live, runtime.domainsFor(taskId)));
+      }
+      return failure;
+    },
+    isCurrent,
+  );
+}
+
+/**
+ * A machine sidecar's own task, at its start and whenever the sidecar comes
+ * back: the proxy's steps in a start's order, the CA and then the pins (the
+ * sidecar's gateway, in this process, is the task's listener, so there is
+ * nothing to launch); with the proxy off, the pins out. A start first voids
+ * any write queued before it. Never fails the task: a failed step leaves it
+ * up without the proxy, its keys withheld, and asks the trust watcher to
+ * retry. Returns what failed, for the caller's log, or null.
+ */
+export async function reconcileSidecarTaskProxy(
+  environment: TaskRootExec,
+  runtime: SecretsProxyRuntime | null,
+  options: { start: boolean },
+): Promise<string | null> {
+  if (!takesTrust(environment)) return null;
+  // A start voids every write queued or under way before it. Whether the
+  // task's hosts view counts is not its to say: task-up admits a task with
+  // no row before recording it, and every other task stands as recorded.
+  if (options.start) invalidateTaskWrites(environment.descriptor.taskId);
+  let failure: unknown = null;
+  try {
+    failure = (await applyTaskRootSteps(environment, { runtime }, async () => undefined))?.error ?? null;
+  } catch (error) {
+    failure = error;
+  }
+  if (failure === null) return null;
+  runtime?.requestTrustDelivery();
+  return `secrets proxy: ${failure instanceof Error ? failure.message : String(failure)}`;
+}
+
+/**
+ * The live path for a change of the task's proxied domains. During a
+ * task-up, task-up is the only routing writer: its end reconciles the then
+ * current domains instead (reconcileTaskProxyState after the start).
+ */
+export function reconcileTaskRouting(environment: Exec, runtime: SecretsProxyRuntime): Promise<void> {
+  if (isTaskStarting(environment.descriptor.taskId)) return Promise.resolve();
+  return reconcileTaskProxyState(environment, runtime);
+}
+
+/** With the proxy off: unpins the task (once; recorded until it pins again). */
+export function unpinTask(environment: Exec, isCurrent?: () => boolean): Promise<void> {
+  return reconcileTaskProxyState(environment, null, isCurrent);
+}

package/lib/secrets-proxy/trust-watch.ts

@@ -0,0 +1,80 @@
+/**
+ * The runner's trust watcher (ADR-153 §3 "The CA"): when the CA bundle
+ * changes (a successor is minted, or the signer retires), put it into every
+ * running loader task the proxy covers (Docker, Apple, a machine's own), so
+ * the processes they start from then on trust both. Runs as the
+ * SecretsProxyRuntime.watchTrust listener, which retries a delivery that
+ * throws.
+ */
+
+import { activeLoaderHostTasks } from "../runtime-state";
+import { reconstructPersistedTaskEnvironment } from "../task-environment/index";
+import type { TaskEnvironmentHandle } from "../task-environment/types";
+import type { SecretsProxyRuntime } from "./runtime";
+import { isProxiedRuntime, reconcileTaskProxyState } from "./task-trust";
+
+export type HostTask = ReturnType<typeof activeLoaderHostTasks>[number];
+
+export interface TrustWatchDeps {
+  tasks(): readonly HostTask[];
+  environmentOf(task: HostTask): Promise<Pick<TaskEnvironmentHandle<unknown>, "exec" | "descriptor"> | null>;
+}
+
+const defaultDeps: TrustWatchDeps = {
+  tasks: activeLoaderHostTasks,
+  environmentOf: reconstructPersistedTaskEnvironment,
+};
+
+/**
+ * Brings every running loader task the proxy covers (Docker, Apple, a
+ * machine's own) to its desired proxy state (reconcileTaskProxyState: the
+ * current CA bundle, the pins for its current domains; or, with `runtime`
+ * null, no pins), one at a time, checking the signal before each. Each task
+ * does only what it lacks. Throws if any task did not get there, or could
+ * not yet (still starting), so the caller retries.
+ */
+async function reconcileRunningTasks(
+  runtime: SecretsProxyRuntime | null,
+  signal: AbortSignal | null,
+  deps: TrustWatchDeps,
+): Promise<void> {
+  const failed: string[] = [];
+  let starting = 0;
+  for (const task of deps.tasks()) {
+    if (signal?.aborted === true) return;
+    if (!isProxiedRuntime(task.environmentProvider)) continue;
+    if (task.statusMirror !== "running") {
+      starting += 1;
+      continue;
+    }
+    try {
+      const environment = await deps.environmentOf(task);
+      if (environment === null) throw new Error("its environment could not be reconstructed");
+      await reconcileTaskProxyState(environment, runtime);
+    } catch (error) {
+      failed.push(`${task.taskId} (${error instanceof Error ? error.message : String(error)})`);
+    }
+  }
+  if (failed.length > 0 || starting > 0) {
+    throw new Error(
+      `the secrets proxy is not in place in every task yet: ${failed.length} failed${failed.length > 0 ? ` (${failed.join("; ")})` : ""}, ` +
+        `${starting} still starting`,
+    );
+  }
+}
+
+/** The SecretsProxyRuntime.watchTrust listener: run when the CA bundle
+ *  changes, at subscription, and as its retries (hourly while it throws). */
+export function reinstallTrustInRunningTasks(
+  runtime: SecretsProxyRuntime,
+  signal: AbortSignal,
+  deps: TrustWatchDeps = defaultDeps,
+): Promise<void> {
+  return reconcileRunningTasks(runtime, signal, deps);
+}
+
+/** With the proxy off: unpins every running Docker task not yet known to
+ *  be unpinned. Run after recovery and hourly while anything is left. */
+export function unpinRunningTasks(deps: TrustWatchDeps = defaultDeps): Promise<void> {
+  return reconcileRunningTasks(null, null, deps);
+}

package/lib/shared-files-s3.ts

@@ -40,6 +40,7 @@ export async function resolveTaskUserIds(
   const fallback = { uid: SHARED_FILES_FALLBACK_UID, gid: SHARED_FILES_FALLBACK_GID };
   try {
     const result = await handle.exec({
+      projectEnvProfile: "maintenance",
       argv: ["/bin/sh", "-c", 'id -u "$1" && id -g "$1"', "ids", user],
       user: "root",
       timeoutMs: 10_000,
@@ -172,6 +173,7 @@ async function mountSharedFilesFromLeaseNow(
   const owner = await resolveTaskUserIds(handle);
   for (const plan of sharedFilesMountPlans(workspacePath, mode, lease)) {
     const result = await handle.exec({
+      projectEnvProfile: "maintenance",
       argv: [
         "/bin/sh",
         "-c",
@@ -224,6 +226,7 @@ export async function unmountSharedFiles(
     await handle
       .exec({
         argv: ["/bin/sh", "-c", 'mountpoint -q "$1" && umount "$1" || true', "umount", directory],
+        projectEnvProfile: "maintenance",
         user: "root",
         timeoutMs: 30_000,
         maxOutputBytes: 4 * 1024,