@moondeck/station 0.0.0-stage → 1.0.0

package/lib/git-identity.ts

@@ -0,0 +1,715 @@
+/**
+ * Set the task creator's git identity inside the container (ADR-029) so commits
+ * are authored as the user who created the task, not the image's generic
+ * identity. Best-effort, mirrors the SSH setup: a failure logs but never aborts
+ * task-up. Name falls back to the email when no display name is known.
+ *
+ * ensureTaskSshIdentity re-asserts the SSH signing identity and its dependent
+ * signing config the same way GitHub tokens are re-asserted at channel ensure.
+ * Git transport is separate: connected users push over HTTPS through gh; SSH
+ * is only the explicit no-connection fallback.
+ */
+
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { eq } from "drizzle-orm";
+
+import { getDb, schema } from "./db";
+import { dockerCli } from "./docker-exec";
+import {
+  removeTaskAppContainer,
+  taskAppContainerName,
+  taskAppContainerState,
+  taskContainerCli,
+} from "./task-container-cli";
+import { onOwnCredentials, whileOnOwnCredentials } from "./cloud-credential-host";
+import { runtimeAuthorityDockerExecArgs } from "./runtime-authority";
+import { RUNTIME_AUTHORITY_ENV } from "./runtime-authority";
+import { assertSafeHostTaskId } from "./task-identity";
+import type { TaskEnvironmentHandle } from "./task-environment/types";
+import {
+  deleteKey,
+  getPublicKey,
+  removeTaskIdentity,
+  writeTaskIdentity,
+} from "./ssh";
+
+const EXEC_TIMEOUT_MS = 10_000;
+
+/** Injectable docker-exec seam (mocked in tests). Async so the docker call
+ *  doesn't block the host event loop (this runs on every channel ensure). */
+export type DockerExec = (
+  args: string[],
+) => Promise<{ status: number | null; stdout?: string; stderr: string }>;
+
+const defaultExec: DockerExec = async (args) => {
+  const res = await taskContainerCli(args, { timeoutMs: EXEC_TIMEOUT_MS });
+  return { status: res.status, stdout: res.stdout, stderr: res.stderr };
+};
+
+function managedTaskExecArgs(
+  user: "node" | "root",
+  container: string,
+  command: string[],
+): string[] {
+  return [
+    "exec",
+    "-u",
+    user,
+    ...runtimeAuthorityDockerExecArgs(),
+    container,
+    ...command,
+  ];
+}
+
+/** A task environment we can exec in (machine or sidecar): the ssh steps run
+ * there as `node` under the runtime authority, key material over stdin. */
+type EnvironmentExec = Pick<TaskEnvironmentHandle, "exec">;
+
+async function environmentStep(
+  environment: EnvironmentExec,
+  argv: [string, ...string[]],
+  stdin?: Uint8Array,
+): Promise<{ status: number | null; stdout: string; stderr: string }> {
+  const result = await environment.exec({
+    projectEnvProfile: "maintenance",
+    argv,
+    user: "node",
+    env: RUNTIME_AUTHORITY_ENV,
+    ...(stdin ? { stdin } : {}),
+    timeoutMs: EXEC_TIMEOUT_MS,
+    maxOutputBytes: 64 * 1024,
+  });
+  return {
+    status: result.exitCode,
+    stdout: Buffer.from(result.stdout).toString("utf8"),
+    stderr: Buffer.from(result.stderr).toString("utf8"),
+  };
+}
+
+export async function setupTaskGitIdentity(
+  taskId: string,
+  name: string | null,
+  email: string | null,
+  exec: DockerExec = defaultExec,
+  environment?: Pick<TaskEnvironmentHandle, "exec">,
+): Promise<boolean> {
+  if (!email) {
+    console.log(`[git] task ${taskId}: no owner email — leaving git identity`);
+    return false;
+  }
+  const displayName = name && name.trim() ? name.trim() : email;
+  const container = taskAppContainerName(taskId);
+  // Values are passed as argv (never interpolated into a shell string), so a
+  // name/email can't shell-inject.
+  const entries: ReadonlyArray<readonly [string, string]> = [
+    ["user.name", displayName],
+    ["user.email", email],
+  ];
+  for (const [key, value] of entries) {
+    const command = ["/usr/bin/git", "config", "--global", key, value] as [
+      string,
+      ...string[],
+    ];
+    const res = environment
+      ? await environment.exec({
+          projectEnvProfile: "maintenance",
+          argv: command,
+          user: "node",
+          env: RUNTIME_AUTHORITY_ENV,
+          timeoutMs: EXEC_TIMEOUT_MS,
+          maxOutputBytes: 64 * 1024,
+        }).then((result) => ({
+          status: result.exitCode,
+          stderr: Buffer.from(result.stderr).toString("utf8"),
+        }))
+      : await exec(managedTaskExecArgs("node", container, command));
+    if (res.status !== 0) {
+      console.warn(
+        `[git] task ${taskId}: git config ${key} failed: ${res.stderr.trim()}`,
+      );
+      return false;
+    }
+  }
+  console.log(`[git] task ${taskId}: git identity → ${displayName} <${email}>`);
+  return true;
+}
+
+// The key's dependent signing config, applied as one unit with the key itself.
+// Deliberately no URL rewrite: doing that here would override a connected
+// user's gh HTTPS credential and make SSH mandatory again.
+//
+// allowed_signers rides along so verification works, not just signing:
+// without gpg.ssh.allowedSignersFile, `git log --format=%G?` prints N ("no
+// signature") for commits that ARE signed and GitHub-verified — git reports
+// the absence of a verification RESULT as the absence of a signature, and a
+// crew read "all 25 commits unsigned" off perfectly signed work (live
+// 2026-08-26). The wildcard principal is deliberate: the task key is the
+// container's only signer, and the identity email may not be configured yet
+// when the key lands.
+export const SSH_SIGNING_CONFIG =
+  "/usr/bin/git config --global gpg.format ssh" +
+  " && /usr/bin/git config --global user.signingkey /home/node/.ssh/id_ed25519.pub" +
+  " && /usr/bin/git config --global commit.gpgsign true" +
+  " && /usr/bin/git config --global tag.gpgsign true" +
+  ' && /usr/bin/printf \'* %s\\n\' "$(/bin/cat /home/node/.ssh/id_ed25519.pub)" > /home/node/.ssh/allowed_signers' +
+  " && /usr/bin/git config --global gpg.ssh.allowedSignersFile /home/node/.ssh/allowed_signers";
+
+export const SSH_TRANSPORT_CONFIG =
+  "/usr/bin/git config --global --unset-all 'url.https://github.com/.insteadOf' >/dev/null 2>&1 || true; " +
+  "/usr/bin/git config --global --replace-all 'url.git@github.com:.insteadOf' 'https://github.com/'; " +
+  '/usr/bin/git config --global core.sshCommand "ssh -o StrictHostKeyChecking=accept-new"';
+
+/** Select SSH transport inside a disconnected task container. */
+export async function configureTaskSshTransport(
+  taskId: string,
+  exec: DockerExec = defaultExec,
+): Promise<boolean> {
+  const res = await exec(managedTaskExecArgs("node", taskAppContainerName(taskId), [
+    "/bin/sh",
+    "-c",
+    SSH_TRANSPORT_CONFIG,
+  ]));
+  if (res.status !== 0) {
+    console.warn(
+      `[ssh] task ${taskId}: SSH transport fallback setup failed: ${res.stderr.trim()}`,
+    );
+    return false;
+  }
+  console.log(`[ssh] task ${taskId}: SSH Git transport fallback selected`);
+  return true;
+}
+
+/**
+ * Copy the SSH identity at `keyPath` into the task container and apply its
+ * dependent git config. Every failure is LOGGED with its step — the silent
+ * `|| true` chain this replaces let a missing key masquerade as a GitHub
+ * permissions problem for an entire afternoon.
+ */
+export async function injectSshIdentity(
+  taskId: string,
+  keyPath: string,
+  exec: DockerExec = defaultExec,
+  environment?: EnvironmentExec,
+): Promise<boolean> {
+  if (environment) {
+    // ADR-121/133: no container and no `docker cp` on a machine — the key
+    // pair rides stdin into files node owns from the start, so there is no
+    // chown race and nothing for the machine's other users to read. Before
+    // this branch every task machine logged `mkdir ~/.ssh failed: spawn
+    // docker ENOENT` on session start and signed nothing (2026-09-10).
+    const privateKey = readFileSync(keyPath);
+    const publicKey = readFileSync(`${keyPath}.pub`);
+    const steps: ReadonlyArray<readonly [string, [string, ...string[]], Uint8Array?]> = [
+      ["mkdir ~/.ssh", ["/bin/sh", "-c", "/bin/mkdir -p /home/node/.ssh && /bin/chmod 700 /home/node/.ssh"]],
+      ["copy key", ["/bin/sh", "-c", "umask 077 && /bin/cat > /home/node/.ssh/id_ed25519"], privateKey],
+      ["copy pubkey", ["/bin/sh", "-c", "umask 022 && /bin/cat > /home/node/.ssh/id_ed25519.pub"], publicKey],
+      ["own + chmod", ["/bin/sh", "-c", "/bin/chmod 700 /home/node/.ssh && /bin/chmod 600 /home/node/.ssh/id_ed25519"]],
+      ["SSH signing config", ["/bin/sh", "-c", SSH_SIGNING_CONFIG]],
+    ];
+    for (const [label, argv, stdin] of steps) {
+      const res = await environmentStep(environment, argv, stdin);
+      if (res.status !== 0) {
+        console.warn(
+          `[ssh] task ${taskId}: ${label} failed: ${res.stderr.trim() || `exit ${res.status}`}`,
+        );
+        return false;
+      }
+    }
+    console.log(`[ssh] task ${taskId}: signing identity + git signing config asserted`);
+    return true;
+  }
+  const container = taskAppContainerName(taskId);
+  const steps: ReadonlyArray<readonly [string, string[]]> = [
+    [
+      "mkdir ~/.ssh",
+      managedTaskExecArgs("root", container, [
+        "/bin/mkdir",
+        "-p",
+        "/home/node/.ssh",
+      ]),
+    ],
+    ["copy key", ["cp", keyPath, `${container}:/home/node/.ssh/id_ed25519`]],
+    ["copy pubkey", ["cp", `${keyPath}.pub`, `${container}:/home/node/.ssh/id_ed25519.pub`]],
+    [
+      "own + chmod",
+      managedTaskExecArgs("root", container, [
+        "/bin/sh", "-c",
+        "/bin/chown -R node:node /home/node/.ssh && /bin/chmod 700 /home/node/.ssh && /bin/chmod 600 /home/node/.ssh/id_ed25519",
+      ]),
+    ],
+    [
+      "SSH signing config",
+      managedTaskExecArgs("node", container, [
+        "/bin/sh",
+        "-c",
+        SSH_SIGNING_CONFIG,
+      ]),
+    ],
+  ];
+  for (const [label, args] of steps) {
+    const res = await exec(args);
+    if (res.status !== 0) {
+      console.warn(
+        `[ssh] task ${taskId}: ${label} failed: ${res.stderr.trim() || `exit ${res.status}`}`,
+      );
+      return false;
+    }
+  }
+  console.log(`[ssh] task ${taskId}: signing identity + git signing config asserted`);
+  return true;
+}
+
+// One in-flight ensure per task. Channel-ensure fires on every reconnect/
+// nudge (seconds apart under load), and `docker cp` is unlink+create — two
+// concurrent injects interleave so one's `cp id_ed25519` lands AFTER the
+// other's chown, leaving the private key owned by the macOS uid and
+// UNREADABLE by node inside the container ("Load key: Permission denied",
+// live 2026-07-21). Serialize, and skip entirely when the key is already
+// good, so the steady state does zero docker work.
+interface SshEnsureInFlight {
+  fingerprint: string;
+  promise: Promise<boolean>;
+}
+
+const sshEnsureInFlight = new Map<string, SshEnsureInFlight>();
+const sshDeleteInFlight = new Map<string, Promise<void>>();
+/** Temporary assignment fence: cleanup seals a task, joins the older ensure,
+ * and deletes the copied key. A later safe assignment may explicitly reopen
+ * the task id. */
+const sealedTaskSshEnsures = new Set<string>();
+/** Durable for this host process: an orphan id is immutable/non-reused, and
+ * no later channel/recovery continuation may materialize its private key. */
+const retiredTaskSshEnsures = new Set<string>();
+
+export async function sealTaskSshIdentityEnsures(
+  taskId: string,
+): Promise<void> {
+  assertSafeHostTaskId(taskId);
+  sealedTaskSshEnsures.add(taskId);
+  for (;;) {
+    const current = sshEnsureInFlight.get(taskId);
+    if (!current) return;
+    await current.promise.catch(() => false);
+    if (sshEnsureInFlight.get(taskId) === current) {
+      sshEnsureInFlight.delete(taskId);
+    }
+  }
+}
+
+export function openTaskSshIdentityEnsures(taskId: string): void {
+  assertSafeHostTaskId(taskId);
+  if (retiredTaskSshEnsures.has(taskId)) {
+    throw new Error(`task ${taskId} has been permanently retired`);
+  }
+  if (sshEnsureInFlight.has(taskId)) {
+    throw new Error(`task ${taskId} still has an SSH identity writer`);
+  }
+  sealedTaskSshEnsures.delete(taskId);
+}
+
+/** Fence future task SSH ensures and join the exact older materialization, if
+ * any. This is stronger than deleting the directory once: channel startup and
+ * runtime recovery intentionally launch SSH reconciliation in the background,
+ * and an older health probe can otherwise resume by writing the key after GC
+ * has already checked absence. Never rejects; strict removal is the final
+ * filesystem authority and will surface any cleanup failure to the cloud. */
+export async function retireTaskSshIdentityEnsures(
+  taskId: string,
+): Promise<void> {
+  assertSafeHostTaskId(taskId);
+  retiredTaskSshEnsures.add(taskId);
+  await sealTaskSshIdentityEnsures(taskId);
+}
+
+function sshPublicKeyFingerprint(publicKey: string): string | null {
+  const [, encoded] = publicKey.trim().split(/\s+/, 3);
+  if (!encoded) return null;
+  try {
+    const material = Buffer.from(encoded, "base64");
+    if (material.length === 0) return null;
+    return createHash("sha256")
+      .update(material)
+      .digest("base64")
+      .replace(/=+$/, "");
+  } catch {
+    return null;
+  }
+}
+
+/** True only when the readable container key is the current DB-authoritative
+ * key for this owner. A copied key is not authority after ssh.key.delete. */
+async function sshKeyHealthy(
+  taskId: string,
+  expectedPublicKey: string,
+  exec: DockerExec,
+  environment?: EnvironmentExec,
+): Promise<boolean> {
+  const probe: [string, ...string[]] = [
+    "/bin/sh",
+    "-c",
+    "test -r /home/node/.ssh/id_ed25519 && /bin/cat /home/node/.ssh/id_ed25519.pub",
+  ];
+  const res = environment
+    ? await environmentStep(environment, probe)
+    : await exec(managedTaskExecArgs("node", taskAppContainerName(taskId), probe));
+  const expectedFingerprint = sshPublicKeyFingerprint(expectedPublicKey);
+  const actualFingerprint = sshPublicKeyFingerprint(res.stdout ?? "");
+  return (
+    res.status === 0 &&
+    expectedFingerprint !== null &&
+    actualFingerprint === expectedFingerprint
+  );
+}
+
+interface EnsureTaskSshIdentityDeps {
+  publicKeyForUser?: (userId: string) => string | null;
+  writeIdentity?: typeof writeTaskIdentity;
+  /** The task's environment when it is a machine or sidecar (no container to
+   * exec into or `docker cp` to). Null/undefined = the container path. */
+  environment?: () => EnvironmentExec | null | undefined;
+}
+
+/**
+ * Ensure the task container has a node-readable SSH signing identity + git
+ * signing config. Idempotent and race-free: skips the copy when the key is already
+ * healthy (the common case), and never runs two injects for one task at
+ * once. Only (re)materializes + copies the key when it is missing or
+ * unreadable. Safe to call on every channel ensure.
+ */
+export async function ensureTaskSshIdentity(
+  taskId: string,
+  ownerUserId: string | null | undefined,
+  exec: DockerExec = defaultExec,
+  deps: EnsureTaskSshIdentityDeps = {},
+): Promise<boolean> {
+  // ADR-154: the runner's own SSH key reaches a task only while the runner
+  // is on its own credentials; leaving them waits for a copy under way.
+  return (
+    (await whileOnOwnCredentials(() => ensureOwnTaskSshIdentity(taskId, ownerUserId, exec, deps))) ?? false
+  );
+}
+
+async function ensureOwnTaskSshIdentity(
+  taskId: string,
+  ownerUserId: string | null | undefined,
+  exec: DockerExec,
+  deps: EnsureTaskSshIdentityDeps,
+): Promise<boolean> {
+  if (
+    retiredTaskSshEnsures.has(taskId) ||
+    sealedTaskSshEnsures.has(taskId)
+  ) {
+    return false;
+  }
+  if (ownerUserId) {
+    const deleting = sshDeleteInFlight.get(ownerUserId);
+    if (deleting) {
+      await deleting.catch(() => {});
+      return ensureTaskSshIdentity(taskId, ownerUserId, exec, deps);
+    }
+  }
+  const publicKeyForUser = deps.publicKeyForUser ?? getPublicKey;
+  const expectedPublicKey = ownerUserId
+    ? publicKeyForUser(ownerUserId)
+    : null;
+  // Consult authority before joining an older in-flight ensure. A deleted key
+  // must turn fallback off immediately even if its prior copy is still readable.
+  if (!ownerUserId || !expectedPublicKey) return false;
+  const expectedFingerprint = sshPublicKeyFingerprint(expectedPublicKey);
+  if (!expectedFingerprint) return false;
+  const existing = sshEnsureInFlight.get(taskId);
+  if (existing) {
+    const result = await existing.promise;
+    if (existing.fingerprint !== expectedFingerprint) {
+      // An older key finished copying while authority rotated. Replace its map
+      // entry and run once more for the current key; never bless A as B.
+      if (sshEnsureInFlight.get(taskId) === existing) {
+        sshEnsureInFlight.delete(taskId);
+      }
+      return ensureTaskSshIdentity(taskId, ownerUserId, exec, deps);
+    }
+    return (
+      result &&
+      sshPublicKeyFingerprint(publicKeyForUser(ownerUserId) ?? "") ===
+        sshPublicKeyFingerprint(expectedPublicKey)
+    );
+  }
+  const pending = ensureTaskSshIdentityOnce(
+    taskId,
+    ownerUserId,
+    expectedPublicKey,
+    exec,
+    deps.writeIdentity ?? writeTaskIdentity,
+    deps.environment?.() ?? undefined,
+  );
+  const entry = { fingerprint: expectedFingerprint, promise: pending };
+  sshEnsureInFlight.set(taskId, entry);
+  try {
+    const result = await pending;
+    return (
+      result &&
+      sshPublicKeyFingerprint(publicKeyForUser(ownerUserId) ?? "") ===
+        sshPublicKeyFingerprint(expectedPublicKey)
+    );
+  } finally {
+    if (sshEnsureInFlight.get(taskId) === entry) {
+      sshEnsureInFlight.delete(taskId);
+    }
+  }
+}
+
+async function ensureTaskSshIdentityOnce(
+  taskId: string,
+  ownerUserId: string,
+  expectedPublicKey: string,
+  exec: DockerExec,
+  writeIdentity: typeof writeTaskIdentity,
+  environment?: EnvironmentExec,
+): Promise<boolean> {
+  // Fast path: the DB-authoritative key is already good → re-assert only the
+  // cheap signing config and skip docker cp churn.
+  if (await sshKeyHealthy(taskId, expectedPublicKey, exec, environment)) {
+    // ADR-154: after the probe too, nothing is written once the runner has
+    // begun leaving its own credentials.
+    if (!onOwnCredentials()) return false;
+    const config: [string, ...string[]] = ["/bin/sh", "-c", SSH_SIGNING_CONFIG];
+    const configured = environment
+      ? await environmentStep(environment, config)
+      : await exec(managedTaskExecArgs("node", taskAppContainerName(taskId), config));
+    return configured.status === 0;
+  }
+
+  // ADR-154: the last hop, fenced: no key is materialized or copied once the
+  // runner has begun leaving its own credentials.
+  if (!onOwnCredentials()) return false;
+  const dir = writeIdentity(taskId, ownerUserId);
+  if (!dir) {
+    console.log(`[ssh] task ${taskId}: no SSH signing identity for its owner`);
+    return false;
+  }
+  const keyPath = resolve(dir, "id_ed25519");
+  try {
+    return await injectSshIdentity(taskId, keyPath, exec, environment);
+  } finally {
+    // Host hygiene (same as task-up): the materialized private key never
+    // outlives the injection.
+    removeTaskIdentity(taskId);
+  }
+}
+
+const REMOVE_MANAGED_SSH_IDENTITY = [
+  "set -eu",
+  "/usr/bin/rm -f -- /home/node/.ssh/id_ed25519 /home/node/.ssh/id_ed25519.pub /home/node/.ssh/allowed_signers",
+  "/usr/bin/git config --global --unset-all gpg.ssh.allowedSignersFile >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --unset-all user.signingkey >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --unset-all commit.gpgsign >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --unset-all tag.gpgsign >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --unset-all gpg.format >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --unset-all 'url.git@github.com:.insteadOf' >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --unset-all core.sshCommand >/dev/null 2>&1 || true",
+  "/usr/bin/git config --global --replace-all 'url.https://github.com/.insteadOf' 'git@github.com:'",
+  "/usr/bin/git config --global --add 'url.https://github.com/.insteadOf' 'ssh://git@github.com/'",
+  "test ! -e /home/node/.ssh/id_ed25519",
+  "test ! -e /home/node/.ssh/id_ed25519.pub",
+  "test -z \"$(/usr/bin/git config --global --get-all user.signingkey 2>/dev/null || true)\"",
+].join("; ");
+
+const SANITIZED_SSH_EXEC_ENV = [
+  "-e",
+  "HOME=/home/node",
+  "-e",
+  "PATH=/usr/bin:/bin",
+  "-e",
+  "XDG_CONFIG_HOME=",
+  "-e",
+  "LD_PRELOAD=",
+  "-e",
+  "LD_LIBRARY_PATH=",
+  "-e",
+  "GIT_CONFIG=",
+  "-e",
+  "GIT_CONFIG_GLOBAL=",
+  "-e",
+  "GIT_CONFIG_SYSTEM=",
+  "-e",
+  "GIT_CONFIG_NOSYSTEM=",
+  "-e",
+  "GIT_CONFIG_COUNT=0",
+  "-e",
+  "GIT_EXEC_PATH=",
+  "-e",
+  "GIT_SSH=",
+  "-e",
+  "GIT_SSH_COMMAND=",
+] as const;
+
+async function checkedSshDocker(
+  exec: DockerExec,
+  args: string[],
+  label: string,
+): Promise<{ status: number | null; stdout?: string; stderr: string }> {
+  const result = await exec(args);
+  if (result.status !== 0) {
+    throw new Error(
+      `${label} failed: ${result.stderr.trim() || `exit ${result.status}`}`,
+    );
+  }
+  return result;
+}
+
+/** Remove a copied SSH key and every config entry that depends on it. A
+ * non-running container is disposable and is removed without executing its
+ * credential-bearing filesystem; Compose recreates it on Resume. */
+export async function removeTaskSshIdentityFromContainer(
+  taskId: string,
+  exec: DockerExec = defaultExec,
+): Promise<void> {
+  const container = taskAppContainerName(taskId);
+  if (exec === defaultExec) {
+    // Backend-aware state/removal: the Apple CLI has no `ps`/`rm` verbs
+    // (review 2026-08-18 round 3, verified live: "Plugin 'container-ps' not
+    // found"), so the shared helpers own the per-backend translation and
+    // the whole-line absence proofs. The injected-exec seam below keeps
+    // scripting docker-shaped answers for tests.
+    const state = await taskAppContainerState(container);
+    if (state.kind === "absent") return;
+    if (state.kind === "unknown") {
+      throw new Error(`inspect ${container} failed: ${state.detail}`);
+    }
+    if (state.kind === "stopped") {
+      // Non-running credential-bearing containers are removed without
+      // executing their filesystem; removal is absence-verified inside.
+      await removeTaskAppContainer(container);
+      return;
+    }
+  } else {
+    const inspectArgs = [
+      "ps",
+      "--all",
+      "--filter",
+      `name=^/${container}$`,
+      "--format",
+      "{{.State}}",
+    ];
+    const listed = await checkedSshDocker(
+      exec,
+      inspectArgs,
+      `inspect ${container}`,
+    );
+    const states = (listed.stdout ?? "")
+      .split(/\r?\n/)
+      .map((state) => state.trim())
+      .filter(Boolean);
+    if (states.length === 0) return;
+    if (states.length !== 1) {
+      throw new Error(
+        `inspect ${container} returned ${states.length} containers`,
+      );
+    }
+    if (states[0] !== "running") {
+      await checkedSshDocker(
+        exec,
+        ["rm", "--force", container],
+        `remove SSH credential-bearing ${container}`,
+      );
+      const remaining = await checkedSshDocker(
+        exec,
+        inspectArgs,
+        `verify removal of ${container}`,
+      );
+      if ((remaining.stdout ?? "").trim()) {
+        throw new Error(`remove ${container} was not verified`);
+      }
+      return;
+    }
+  }
+
+  await checkedSshDocker(
+    exec,
+    [
+      "exec",
+      "-u",
+      "node",
+      ...SANITIZED_SSH_EXEC_ENV,
+      ...runtimeAuthorityDockerExecArgs(),
+      container,
+      "/usr/bin/env",
+      "-i",
+      "HOME=/home/node",
+      "PATH=/usr/bin:/bin",
+      "/bin/sh",
+      "-c",
+      REMOVE_MANAGED_SSH_IDENTITY,
+    ],
+    `remove managed SSH identity from ${container}`,
+  );
+}
+
+function sshCredentialTaskIdsForUser(userId: string): string[] {
+  return getDb()
+    .select({
+      taskId: schema.hostTasks.taskId,
+      endedAt: schema.hostTasks.endedAt,
+      composeProject: schema.hostTasks.composeProject,
+      statusMirror: schema.hostTasks.statusMirror,
+    })
+    .from(schema.hostTasks)
+    .where(eq(schema.hostTasks.ownerUserId, userId))
+    .all()
+    .filter(
+      (task) =>
+        task.endedAt == null ||
+        task.composeProject != null ||
+        task.statusMirror === "error",
+    )
+    .map((task) => task.taskId);
+}
+
+export interface DeleteUserSshIdentityDeps {
+  /** Synchronous fail-closed check that must pass before authority changes. */
+  preflight?: () => void;
+  deleteStored?: (userId: string) => void;
+  taskIds?: (userId: string) => string[];
+  removeFromContainer?: (taskId: string) => Promise<void>;
+}
+
+/** Serialize deletion against new ensures, drain every older ensure, scrub all
+ * copied material, and only then delete authority. A failed scrub leaves the
+ * key authoritative and retryable instead of creating an unrecoverable split
+ * where host authority is gone but a valid private key remains in a container. */
+export async function deleteUserSshIdentity(
+  userId: string,
+  deps: DeleteUserSshIdentityDeps = {},
+): Promise<void> {
+  const existing = sshDeleteInFlight.get(userId);
+  if (existing) return existing;
+  const run = async (): Promise<void> => {
+    deps.preflight?.();
+    const taskIds = (deps.taskIds ?? sshCredentialTaskIdsForUser)(userId);
+    const removeFromContainer =
+      deps.removeFromContainer ?? removeTaskSshIdentityFromContainer;
+    const settled = await Promise.allSettled(
+      taskIds.map(async (taskId) => {
+        await sshEnsureInFlight.get(taskId)?.promise.catch(() => {});
+        removeTaskIdentity(taskId);
+        await removeFromContainer(taskId);
+      }),
+    );
+    const failed = settled.find(
+      (result): result is PromiseRejectedResult => result.status === "rejected",
+    );
+    if (failed) throw failed.reason;
+    (deps.deleteStored ?? deleteKey)(userId);
+  };
+  let tracked!: Promise<void>;
+  tracked = run().finally(() => {
+    if (sshDeleteInFlight.get(userId) === tracked) {
+      sshDeleteInFlight.delete(userId);
+    }
+  });
+  sshDeleteInFlight.set(userId, tracked);
+  return tracked;
+}