@runuai/host 0.9.150 → 0.9.151

package/lib/credential-store.ts

@@ -10,18 +10,34 @@
  * null as "credentials stay on this machine".
  *
  * Secret material never travels on argv where `ps` would show it: a put
- * writes its request to a 0600 file in a fresh 0700 temp dir and hands the CLI
+ * writes its request to a 0600 file in a fresh 0700 dir and hands the CLI
  * `--cli-input-json file://<path>`, removed as soon as the CLI exits. (Not
  * `file:///dev/stdin`: the CLI opens the file twice and a pipe reads empty the
  * second time — verified against aws-cli 2.33.) Nothing here logs a value.
+ *
+ * That dir is on a memory-backed filesystem (tmpfs), never a disk: a process
+ * killed between the write and its cleanup leaves the file in memory only,
+ * gone at the next boot, and the next process to open a store removes any a
+ * dead process left. With no memory-backed dir to use, a write is refused
+ * rather than staged on disk.
  */
 
 import { spawn } from "node:child_process";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import {
+  lstatSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  statfsSync,
+  statSync,
+  writeFileSync,
+  type Stats,
+} from "node:fs";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 
 import { env } from "./env";
+import { assertNoAliases } from "./fs-aliases";
 import { leaseAwsEnv } from "./lease";
 
 const DEFAULT_TIMEOUT_MS = 30_000;
@@ -51,6 +67,29 @@ export class ImmutableParameterConflictError extends Error {
 }
 
 /** A failed/ambiguous create could not yet be proved to be our prior write. */
+/** A read a caller's deadline ends: the AWS CLI is killed when its signal
+ *  fires, and the read fails (the caller checks `signal.aborted`). */
+export interface StoreCallOptions {
+  signal?: AbortSignal;
+}
+
+export interface ImmutablePutOptions {
+  /**
+   * Ends the write where it is: the AWS CLI is killed, so nothing is retried
+   * past it. A request already sent may still have been applied; the caller
+   * treats an aborted write as unknown, never as absent.
+   */
+  signal?: AbortSignal;
+}
+
+/** The write was stopped by its caller's signal; it may or may not exist. */
+export class ImmutableParameterAbortedError extends Error {
+  constructor(message = "immutable parameter write was stopped before it finished") {
+    super(message);
+    this.name = "ImmutableParameterAbortedError";
+  }
+}
+
 export class ImmutableParameterUnverifiedError extends Error {
   constructor(message = "immutable parameter create could not be verified") {
     super(message);
@@ -74,9 +113,10 @@ export interface CredentialStore {
     name: string,
     value: string,
     tags: Record<string, string>,
+    options?: ImmutablePutOptions,
   ): Promise<ImmutableParameterPutResult>;
   /** The credential's plaintext, or null when it does not exist. */
-  get(name: string): Promise<string | null>;
+  get(name: string, options?: StoreCallOptions): Promise<string | null>;
   /**
    * One exact immutable parameter version. Project-env replica migration
    * uses this for legacy value references: reading the latest value and then
@@ -109,7 +149,7 @@ export interface CredentialStore {
   /** The tags on one parameter. Metadata only: `ssm:ListTagsForResource`
    *  reveals no value and needs no KMS grant — the sign-in box (ADR-136)
    *  rebuilds an account's label from here without ever reading the secret. */
-  listTags(name: string): Promise<Record<string, string>>;
+  listTags(name: string, options?: StoreCallOptions): Promise<Record<string, string>>;
   /** Every parameter NAME under `path` (recursive), no values — the
    *  `describe-parameters` listing that is allowed on `*` and never decrypts. */
   describeNames(path: string): Promise<string[]>;
@@ -124,8 +164,8 @@ export interface CliResult {
   stderr: string;
 }
 
-/** `aws --region <r> --output json <args>`. */
-export type AwsCliRunner = (args: string[]) => Promise<CliResult>;
+/** `aws --region <r> --output json <args>`. A signal kills the process. */
+export type AwsCliRunner = (args: string[], options?: { signal?: AbortSignal }) => Promise<CliResult>;
 
 export function isParameterName(name: string): boolean {
   return PARAMETER_NAME.test(name);
@@ -151,11 +191,16 @@ export function parameterName(prefix: string, ...segments: string[]): string {
 }
 
 export function defaultAwsCliRunner(region: string): AwsCliRunner {
-  return (args) =>
+  return (args, options = {}) =>
     new Promise<CliResult>((resolve) => {
       let stdout = "";
       let stderr = "";
       let settled = false;
+      const { signal } = options;
+      if (signal?.aborted) {
+        resolve({ status: null, stdout: "", stderr: "aborted before it started" });
+        return;
+      }
       const child = spawn("aws", ["--region", region, "--output", "json", ...args], {
         stdio: ["ignore", "pipe", "pipe"],
         // ADR-133: a task machine acts under its lease (empty elsewhere — an
@@ -166,6 +211,7 @@ export function defaultAwsCliRunner(region: string): AwsCliRunner {
         if (settled) return;
         settled = true;
         clearTimeout(timer);
+        signal?.removeEventListener("abort", abort);
         resolve({ status, stdout, stderr });
       };
       const timer = setTimeout(() => {
@@ -174,6 +220,13 @@ export function defaultAwsCliRunner(region: string): AwsCliRunner {
         grace.unref?.();
       }, DEFAULT_TIMEOUT_MS);
       timer.unref?.();
+      // The caller has moved on: the process dies now, and whatever it had
+      // done is unknown to it, so nothing waits for its output.
+      const abort = (): void => {
+        child.kill("SIGKILL");
+        settle(null);
+      };
+      signal?.addEventListener("abort", abort, { once: true });
       child.stdout?.setEncoding("utf8");
       child.stderr?.setEncoding("utf8");
       child.stdout?.on("data", (chunk: string) => (stdout += chunk));
@@ -183,22 +236,240 @@ export function defaultAwsCliRunner(region: string): AwsCliRunner {
     });
 }
 
-/** Run `work` with a 0600 file holding `content`, inside a private temp dir
- *  that is removed afterwards whatever happens. */
+/** `statfs` f_type of tmpfs (linux/magic.h). */
+const TMPFS_MAGIC = 0x01021994;
+const STAGING_PREFIX = "uai-credential-store-";
+
+export interface StagingDeps {
+  statfs?: (path: string) => { type: number | bigint };
+  candidates?: Array<string | undefined>;
+}
+
+/**
+ * The memory-backed dir a write's request (its value included) is staged in
+ * while the CLI reads it: the person's runtime dir first (0700, tmpfs under
+ * systemd), then /dev/shm. Null when none here is tmpfs.
+ */
+export function memoryBackedDir(deps: StagingDeps = {}): string | null {
+  const statfs = deps.statfs ?? ((path: string) => statfsSync(path));
+  for (const dir of deps.candidates ?? [process.env.XDG_RUNTIME_DIR, "/dev/shm", "/run/shm"]) {
+    if (!dir) continue;
+    try {
+      if (Number(statfs(dir).type) === TMPFS_MAGIC) return dir;
+    } catch {
+      // Absent here: the next candidate.
+    }
+  }
+  return null;
+}
+
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // EPERM: alive, someone else's.
+    return (err as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+/**
+ * Remove what a dead process left staged: in `root`, a dir naming a process
+ * that is gone; in the old on-disk temp dir, any staged dir from before this
+ * moved to memory, once older than a CLI call could be. Never a live
+ * process's.
+ */
+export function scrubStagedWrites(
+  root: string,
+  legacyRoot: string | null = tmpdir(),
+  now: number = Date.now(),
+): { removed: number; legacyDueAt: number | null } {
+  let removed = 0;
+  for (const name of safeList(root)) {
+    const match = OWN_STAGED.exec(name);
+    if (!match) continue;
+    const pid = Number(match[1]);
+    if (pid === process.pid || processAlive(pid)) continue;
+    rmSync(join(root, name), { recursive: true, force: true });
+    removed += 1;
+  }
+  if (!legacyRoot || legacyRoot === root) return { removed, legacyDueAt: null };
+  const legacy = scrubLegacyStagedWrites(legacyRoot, now);
+  return { removed: removed + legacy.removed, legacyDueAt: legacy.dueAt };
+}
+
+/** A staged dir this build names: by the pid of the process it belongs to. */
+const OWN_STAGED = new RegExp(`^${STAGING_PREFIX}(\\d+)-`);
+/** Longer than any CLI call: a staged dir of a build from before writes
+ *  moved to memory, this old, belongs to nothing still running. */
+export const LEGACY_STAGED_MAX_AGE_MS = 10 * 60_000;
+
+/** Remove the old build's staged dirs past LEGACY_STAGED_MAX_AGE_MS; `dueAt`
+ *  is when the youngest one left will be, or null when none is left. */
+function scrubLegacyStagedWrites(legacyRoot: string, now: number): { removed: number; dueAt: number | null } {
+  let removed = 0;
+  let dueAt: number | null = null;
+  for (const name of safeList(legacyRoot)) {
+    if (!name.startsWith(STAGING_PREFIX) || OWN_STAGED.test(name)) continue;
+    let modified: number;
+    try {
+      modified = statSync(join(legacyRoot, name)).mtimeMs;
+    } catch {
+      continue;
+    }
+    if (now - modified < LEGACY_STAGED_MAX_AGE_MS) {
+      dueAt = Math.min(dueAt ?? Infinity, modified + LEGACY_STAGED_MAX_AGE_MS);
+      continue;
+    }
+    rmSync(join(legacyRoot, name), { recursive: true, force: true });
+    removed += 1;
+  }
+  return { removed, dueAt };
+}
+
+/** Come back for the old build's staged dirs too young to remove yet: a
+ *  write killed just before this process started is still removed, once
+ *  nothing can be using it. */
+function scheduleLegacyScrub(legacyRoot: string, dueAt: number): void {
+  const timer = setTimeout(() => {
+    const next = scrubLegacyStagedWrites(legacyRoot, Date.now()).dueAt;
+    if (next !== null) scheduleLegacyScrub(legacyRoot, next);
+  }, Math.max(0, dueAt - Date.now()) + 1_000);
+  timer.unref?.();
+}
+
+export interface StagedWritesLeftBehind {
+  removed: string[];
+  /** A write that may be under way: this process's own still staged, or
+   *  another live process's or an older build's too young to tell. */
+  pending: string[];
+  failed: Array<{ path: string; error: unknown }>;
+}
+
+/**
+ * ADR-154 §13, the cutover purge: remove every staged write in `roots`,
+ * this build's or an older one's, that nothing can still be using.
+ *
+ * This process knows its own: a write under way is staged
+ * (`withPrivateInputFile`), and anything else named for it is stale, left by
+ * a write whose cleanup failed. Another live process's write is under way
+ * while it is younger than any CLI call can last (LEGACY_STAGED_MAX_AGE_MS),
+ * and its writer removes it; past that age it is a leftover whatever its
+ * pid now names (a reused pid is not the writer). An older build's names no
+ * process. A write that may be under way is reported pending, for a later
+ * call: the purge is not done while one is. Another user's entry is never
+ * ours. Every entry is tried. Reported, not thrown: a root that cannot be
+ * listed or is not there (what it holds is unknown), an entry that is a
+ * link, holds one leading out of it or a file with a second name (removing
+ * it would keep what it holds under the other), and a failed removal.
+ */
+export function removeStagedWritesLeftBehind(
+  roots: readonly string[],
+  now: number = Date.now(),
+): StagedWritesLeftBehind {
+  const uid = process.getuid?.();
+  const result: StagedWritesLeftBehind = { removed: [], pending: [], failed: [] };
+  for (const root of new Set(roots)) {
+    let names: string[];
+    try {
+      names = readdirSync(root);
+    } catch (error) {
+      // Not there proves nothing either: a root this runner stages in is
+      // always there, unless something unmounted or removed it.
+      result.failed.push({ path: root, error });
+      continue;
+    }
+    for (const name of names) {
+      if (!name.startsWith(STAGING_PREFIX)) continue;
+      const path = join(root, name);
+      let stat: Stats;
+      try {
+        stat = lstatSync(path);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "ENOENT") result.failed.push({ path, error });
+        continue;
+      }
+      if (uid !== undefined && stat.uid !== uid) continue;
+      const young = now - stat.mtimeMs < LEGACY_STAGED_MAX_AGE_MS;
+      const own = OWN_STAGED.exec(name);
+      const mayBeUnderWay = own
+        ? Number(own[1]) === process.pid
+          ? stagedInUse.has(resolve(path))
+          : young && processAlive(Number(own[1]))
+        : young;
+      if (mayBeUnderWay) {
+        result.pending.push(path);
+        continue;
+      }
+      // Removed by this name only when no other keeps what it holds.
+      if (stat.isSymbolicLink()) {
+        result.failed.push({
+          path,
+          error: new Error("a symbolic link where a write is staged; neither it nor what it leads to is removed"),
+        });
+        continue;
+      }
+      try {
+        assertNoAliases(path);
+      } catch (error) {
+        result.failed.push({ path, error });
+        continue;
+      }
+      try {
+        rmSync(path, { recursive: true, force: true });
+        result.removed.push(path);
+      } catch (error) {
+        result.failed.push({ path, error });
+      }
+    }
+  }
+  return result;
+}
+
+function safeList(dir: string): string[] {
+  try {
+    return readdirSync(dir);
+  } catch {
+    return [];
+  }
+}
+
+/** This process's staged writes under way, by their directory. */
+const stagedInUse = new Set<string>();
+
+/** Scrubbed once per process per root, when its first store is made. */
+const scrubbed = new Set<string>();
+
+/** Run `work` with a 0600 file holding `content`, inside a private dir under
+ *  `root` (memory-backed) that is removed afterwards whatever happens. */
 async function withPrivateInputFile<T>(
+  root: string | null,
   content: string,
   work: (path: string) => Promise<T>,
 ): Promise<T> {
-  const dir = mkdtempSync(join(tmpdir(), "uai-credential-store-"));
+  // Fail closed: a credential is never staged on disk.
+  if (root === null) throw new Error("no memory-backed directory to stage a credential write in");
+  const dir = mkdtempSync(join(root, `${STAGING_PREFIX}${process.pid}-`));
+  stagedInUse.add(resolve(dir));
   try {
     const path = join(dir, "input.json");
     writeFileSync(path, content, { mode: 0o600 });
     return await work(path);
   } finally {
     rmSync(dir, { recursive: true, force: true });
+    stagedInUse.delete(resolve(dir));
   }
 }
 
+/** Tests only: stage `content` as a store write does, around `work`. */
+export function withStagedWriteForTests<T>(
+  root: string,
+  content: string,
+  work: (path: string) => Promise<T>,
+): Promise<T> {
+  return withPrivateInputFile(root, content, work);
+}
+
 function isNotFound(result: CliResult): boolean {
   return /ParameterNotFound/.test(result.stderr);
 }
@@ -220,6 +491,20 @@ function checkedTagEntries(tags: Record<string, string>): Array<[string, string]
   return entries;
 }
 
+/** A CLI failure: the AWS error code (a fixed vocabulary) and exit status
+ *  ride as fields a log may print; the message keeps stderr's first line. */
+export class CredentialStoreCommandError extends Error {
+  constructor(
+    message: string,
+    readonly op: string,
+    readonly status: number | null,
+    readonly awsCode: string | null,
+  ) {
+    super(message);
+    this.name = "CredentialStoreCommandError";
+  }
+}
+
 /** Summarize a CLI failure WITHOUT echoing stdout (which could carry a value). */
 function failure(op: string, result: CliResult): Error {
   const line =
@@ -227,13 +512,39 @@ function failure(op: string, result: CliResult): Error {
       .split("\n")
       .map((l) => l.trim())
       .find((l) => l.length > 0) ?? (result.status === null ? "spawn failed or timed out" : "");
-  return new Error(`aws ssm ${op} failed (exit ${result.status ?? "none"}): ${line}`);
+  const awsCode = /An error occurred \(([A-Za-z0-9]+)\)/.exec(result.stderr)?.[1] ?? null;
+  return new CredentialStoreCommandError(
+    `aws ssm ${op} failed (exit ${result.status ?? "none"}): ${line}`,
+    op,
+    result.status,
+    awsCode,
+  );
+}
+
+/**
+ * A store failure as a log may name it: a category, never a message, which
+ * can carry what AWS or another dependency said (or echoed back).
+ */
+export function storeFailureKind(err: unknown): string {
+  if (err instanceof ImmutableParameterConflictError) return "conflict";
+  if (err instanceof ImmutableParameterUnverifiedError) return "unverified";
+  if (err instanceof ImmutableParameterAbortedError) return "aborted";
+  if (err instanceof CredentialStoreCommandError) {
+    return `${err.op}: ${err.awsCode ?? `exit ${err.status ?? "none"}`}`;
+  }
+  return err instanceof Error ? err.name : "unknown";
 }
 
 export interface AwsSsmCredentialStoreOptions {
   region: string;
   kmsKeyId: string;
   runner?: AwsCliRunner;
+  /** Tests only: where writes are staged instead of a memory-backed dir;
+   *  null as if this machine had none. */
+  stagingDir?: string | null;
+  /** Tests only: where an older build staged writes on disk (the temp dir
+   *  by default; none when `stagingDir` is given). */
+  legacyStagingDir?: string | null;
 }
 
 export class AwsSsmCredentialStore implements CredentialStore {
@@ -241,9 +552,25 @@ export class AwsSsmCredentialStore implements CredentialStore {
   private readonly run: AwsCliRunner;
   private readonly kmsKeyId: string;
 
+  private readonly stagingRoot: string | null;
+
   constructor(opts: AwsSsmCredentialStoreOptions) {
     this.run = opts.runner ?? defaultAwsCliRunner(opts.region);
     this.kmsKeyId = opts.kmsKeyId;
+    this.stagingRoot = opts.stagingDir !== undefined ? opts.stagingDir : memoryBackedDir();
+    if (this.stagingRoot !== null && !scrubbed.has(this.stagingRoot)) {
+      scrubbed.add(this.stagingRoot);
+      scrubStagedWrites(this.stagingRoot, null);
+    }
+    // An older build's plaintext on disk goes whether or not this host can
+    // stage in memory: one that now refuses every write may still hold it.
+    const legacyRoot =
+      opts.legacyStagingDir !== undefined ? opts.legacyStagingDir : opts.stagingDir !== undefined ? null : tmpdir();
+    if (legacyRoot !== null && legacyRoot !== this.stagingRoot && !scrubbed.has(`legacy:${legacyRoot}`)) {
+      scrubbed.add(`legacy:${legacyRoot}`);
+      const { dueAt } = scrubLegacyStagedWrites(legacyRoot, Date.now());
+      if (dueAt !== null) scheduleLegacyScrub(legacyRoot, dueAt);
+    }
   }
 
   async put(name: string, value: string): Promise<void> {
@@ -259,7 +586,7 @@ export class AwsSsmCredentialStore implements CredentialStore {
       Tier: "Advanced",
       Overwrite: true,
     });
-    const result = await withPrivateInputFile(input, (path) =>
+    const result = await withPrivateInputFile(this.stagingRoot, input, (path) =>
       this.run(["ssm", "put-parameter", "--cli-input-json", `file://${path}`]),
     );
     if (result.status !== 0) throw failure("put-parameter", result);
@@ -269,12 +596,15 @@ export class AwsSsmCredentialStore implements CredentialStore {
     name: string,
     value: string,
     tags: Record<string, string>,
+    options: ImmutablePutOptions = {},
   ): Promise<ImmutableParameterPutResult> {
     if (!isParameterName(name)) throw new Error(`invalid parameter name: ${name}`);
     const entries = checkedTagEntries(tags);
     if (entries.length === 0) {
       throw new Error("immutable parameter identity tags are required");
     }
+    const { signal } = options;
+    if (signal?.aborted) throw new ImmutableParameterAbortedError();
     const input = JSON.stringify({
       Name: name,
       Value: value,
@@ -284,18 +614,20 @@ export class AwsSsmCredentialStore implements CredentialStore {
       Overwrite: false,
       Tags: entries.map(([Key, Value]) => ({ Key, Value })),
     });
-    const result = await withPrivateInputFile(input, (path) =>
-      this.run(["ssm", "put-parameter", "--cli-input-json", `file://${path}`]),
+    const result = await withPrivateInputFile(this.stagingRoot, input, (path) =>
+      this.run(["ssm", "put-parameter", "--cli-input-json", `file://${path}`], { signal }),
     );
     if (result.status === 0) return "created";
+    if (signal?.aborted) throw new ImmutableParameterAbortedError();
 
     // The CLI can lose the successful response, and a retry then receives
     // ParameterAlreadyExists. The sign-in-box identity is intentionally
     // write-only, so authenticate the prior write using metadata only.
     let observed: Record<string, string>;
     try {
-      observed = await this.listTags(name);
+      observed = await this.tagsOf(name, signal);
     } catch {
+      if (signal?.aborted) throw new ImmutableParameterAbortedError();
       throw new ImmutableParameterUnverifiedError();
     }
     let missing = false;
@@ -331,15 +663,11 @@ export class AwsSsmCredentialStore implements CredentialStore {
     if (result.status !== 0) throw failure("add-tags-to-resource", result);
   }
 
-  async get(name: string): Promise<string | null> {
+  async get(name: string, options: StoreCallOptions = {}): Promise<string | null> {
     if (!isParameterName(name)) throw new Error(`invalid parameter name: ${name}`);
-    const result = await this.run([
-      "ssm",
-      "get-parameter",
-      "--name",
-      name,
-      "--with-decryption",
-    ]);
+    const result = await this.run(["ssm", "get-parameter", "--name", name, "--with-decryption"], {
+      signal: options.signal,
+    });
     if (result.status !== 0) {
       if (isNotFound(result)) return null;
       throw failure("get-parameter", result);
@@ -443,16 +771,16 @@ export class AwsSsmCredentialStore implements CredentialStore {
     );
   }
 
-  async listTags(name: string): Promise<Record<string, string>> {
+  async listTags(name: string, options: StoreCallOptions = {}): Promise<Record<string, string>> {
+    return this.tagsOf(name, options.signal);
+  }
+
+  private async tagsOf(name: string, signal?: AbortSignal): Promise<Record<string, string>> {
     if (!isParameterName(name)) throw new Error(`invalid parameter name: ${name}`);
-    const result = await this.run([
-      "ssm",
-      "list-tags-for-resource",
-      "--resource-type",
-      "Parameter",
-      "--resource-id",
-      name,
-    ]);
+    const result = await this.run(
+      ["ssm", "list-tags-for-resource", "--resource-type", "Parameter", "--resource-id", name],
+      { signal },
+    );
     if (result.status !== 0) throw failure("list-tags-for-resource", result);
     const parsed = JSON.parse(result.stdout || "{}") as {
       TagList?: Array<{ Key?: unknown; Value?: unknown }> | null;