@zgeoff/atc 3.2.0 → 3.3.0

package/src/run-daemon-restart-worker.ts

@@ -0,0 +1,77 @@
+import { mkdirSync } from 'node:fs';
+import { buildRestartFailure } from './build-restart-failure';
+import { formatRestartReport } from './format-restart-report';
+import type { RestartResult } from './parse-restart-result';
+import { readRestartResult } from './read-restart-result';
+import { removeRestartResult } from './remove-restart-result';
+import { restartDaemon } from './restart-daemon';
+import type { RestartOptions } from './restart-daemon';
+import { claimDaemonLock } from './shared/claim-daemon-lock';
+import { restartLockFile, restartsDir } from './shared/config';
+import { writeRestartResult } from './write-restart-result';
+
+// How long a restart that finds another in flight waits for it to finish.
+const JOIN_WAIT_MS = 15 * 60_000;
+
+/**
+ * Runs one restart as the handoff worker: it takes the restart lock and
+ * restarts the daemon, or, when another restart holds the lock, waits for
+ * that restart to finish and reports its result without touching the
+ * daemon. Prints each step to stdout, which the caller follows in the run
+ * log, and ends the log with the result record. Resolves the exit code.
+ */
+export async function runDaemonRestartWorker(options: RestartOptions): Promise<number> {
+  console.log(`worker pid ${process.pid}`);
+
+  mkdirSync(restartsDir, { recursive: true });
+
+  const lock = await claimDaemonLock(restartLockFile, 0);
+
+  let result: RestartResult;
+
+  if (lock === null) {
+    result = await waitForRestartInFlight(options.runID);
+  } else {
+    try {
+      // A joiner reads last.json once it gets the lock, so a result left by
+      // an earlier restart must not outlive this run's start.
+      removeRestartResult();
+
+      result = await restartDaemon(options).catch((error: unknown) => {
+        const reason = error instanceof Error ? error.message : String(error);
+
+        return buildRestartFailure(options.runID, reason);
+      });
+
+      writeRestartResult(result);
+    } finally {
+      lock.dispose();
+    }
+  }
+
+  for (const line of formatRestartReport(result)) {
+    console.log(line);
+  }
+
+  console.log(JSON.stringify(result));
+
+  return result.code;
+}
+
+async function waitForRestartInFlight(runID: string): Promise<RestartResult> {
+  console.log('another restart is in flight; this run joins it and reports its result');
+
+  const held = await claimDaemonLock(restartLockFile, JOIN_WAIT_MS);
+
+  if (held === null) {
+    return buildRestartFailure(runID, 'timed out waiting for the restart in flight to finish');
+  }
+
+  const finished = readRestartResult();
+
+  held.dispose();
+
+  return finished === null
+    ? buildRestartFailure(runID, 'the restart in flight left no result')
+    : { ...finished, runID };
+}

package/src/run-daemon-restart.ts

@@ -0,0 +1,228 @@
+import {
+  closeSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  statSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { collectRestartEnv } from './collect-restart-env';
+import { collectRestartPlan } from './collect-restart-plan';
+import { formatRestartPreflight } from './format-restart-preflight';
+import { parseRestartResult } from './parse-restart-result';
+import { buildATCCommand } from './shared/build-atc-command';
+import { restartsDir } from './shared/config';
+import { isProcessAlive } from './shared/is-process-alive';
+import { spawnATCDetached } from './shared/spawn-atc-detached';
+
+interface DaemonRestartOptions {
+  readonly listen: string | null;
+  readonly tokenFile: string | null;
+  readonly timeoutSeconds: number | null;
+  readonly dryRun: boolean;
+}
+
+const LOG_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+// How long the follower waits for the worker to report its pid before it
+// gives up on a worker that never started.
+const WORKER_START_WAIT_MS = 30_000;
+
+/**
+ * Runs `atc daemon restart` in the caller: a dry run prints the preflight
+ * and stops, and a restart hands the work to a detached worker, because the
+ * restart can end the process that asked for it. The worker writes a run
+ * log, which this follows line by line until the worker's result record
+ * ends it. Resolves the exit code: the worker's, or 1 when the worker
+ * vanishes without a result.
+ */
+export async function runDaemonRestart(options: DaemonRestartOptions): Promise<number> {
+  const callerSession = process.env['ATC_SESSION_ID'] ?? null;
+
+  const plan = await collectRestartPlan(callerSession);
+
+  if (options.dryRun) {
+    for (const line of formatRestartPreflight(plan)) {
+      console.log(line);
+    }
+
+    return 0;
+  }
+
+  mkdirSync(restartsDir, { recursive: true });
+  removeStaleRestartLogs();
+
+  const runID = mintRunID();
+  const logPath = join(restartsDir, `${runID}.log`);
+  const workerArgs = buildWorkerArgs(runID, callerSession, options);
+  const logFD = openSync(logPath, 'a');
+  let workerPID: number | null = null;
+
+  try {
+    if (plan.replacement.kind === 'unit') {
+      const started = await startWorkerInUnit(runID, logPath, workerArgs);
+
+      if (started !== 0) {
+        console.error(`atc daemon restart: systemd-run failed with exit code ${started}`);
+
+        return 1;
+      }
+    } else {
+      workerPID =
+        spawnATCDetached(workerArgs, {
+          env: collectRestartEnv(process.env),
+          stdio: ['ignore', logFD, logFD],
+        }).pid ?? null;
+    }
+  } finally {
+    closeSync(logFD);
+  }
+
+  console.log(`progress: ${logPath}`);
+
+  return printRunLog(runID, logPath, workerPID);
+}
+
+function mintRunID(): string {
+  return `${Date.now().toString(36)}-${crypto.randomUUID().slice(0, 6)}`;
+}
+
+function buildWorkerArgs(
+  runID: string,
+  callerSession: string | null,
+  options: DaemonRestartOptions,
+): string[] {
+  return [
+    'daemon',
+    'restart-worker',
+    runID,
+    ...(callerSession === null ? [] : ['--session', callerSession]),
+    ...(options.listen === null ? [] : ['--listen', options.listen]),
+    ...(options.tokenFile === null ? [] : ['--token-file', options.tokenFile]),
+    ...(options.timeoutSeconds === null ? [] : ['--timeout', String(options.timeoutSeconds)]),
+  ];
+}
+
+// A restart through a unit stops the unit's whole cgroup, which holds a
+// hosted caller and its children, so the worker starts as a transient unit
+// of its own. It gets HOME, XDG_RUNTIME_DIR, PATH and every ATC_ variable
+// explicitly, since a transient unit otherwise inherits the user manager's
+// environment and with it the real state directory.
+function startWorkerInUnit(
+  runID: string,
+  logPath: string,
+  workerArgs: readonly string[],
+): Promise<number> {
+  const env = collectRestartEnv(process.env);
+
+  const setenvs = Object.entries(env)
+    .filter(
+      ([key]) =>
+        key === 'HOME' || key === 'XDG_RUNTIME_DIR' || key === 'PATH' || key.startsWith('ATC_'),
+    )
+    .map(([key, value]) => `--setenv=${key}=${value}`);
+
+  const proc = Bun.spawn(
+    [
+      'systemd-run',
+      '--user',
+      '--collect',
+      '--quiet',
+      '--unit',
+      `atc-daemon-restart-${runID}`,
+      `--property=StandardOutput=append:${logPath}`,
+      `--property=StandardError=append:${logPath}`,
+      ...setenvs,
+      '--',
+      ...buildATCCommand(workerArgs),
+    ],
+    { env, stdin: 'ignore', stdout: 'ignore', stderr: 'inherit' },
+  );
+
+  return proc.exited;
+}
+
+async function printRunLog(
+  runID: string,
+  logPath: string,
+  workerPIDAtStart: number | null,
+): Promise<number> {
+  const startedAt = Date.now();
+  let workerPID = workerPIDAtStart;
+  let printed = 0;
+  let pending = '';
+  let vanished = false;
+
+  for (;;) {
+    const text = readLog(logPath);
+
+    pending += text.slice(printed);
+    printed = text.length;
+
+    const lines = pending.split('\n');
+
+    pending = lines.pop() ?? '';
+
+    for (const line of lines) {
+      const result = parseRestartResult(line);
+
+      if (result !== null && result.runID === runID) {
+        return result.code;
+      }
+
+      const pidMatch = /^worker pid (?<pid>\d+)$/.exec(line);
+
+      if (pidMatch !== null) {
+        workerPID = Number(pidMatch.groups?.['pid']);
+      }
+
+      console.log(line);
+    }
+
+    if (vanished) {
+      return printVanishedWorker(logPath);
+    }
+
+    // One more read after the worker is gone, in case its last lines landed
+    // between the read above and its exit.
+    vanished =
+      (workerPID !== null && !isProcessAlive(workerPID)) ||
+      (workerPID === null && Date.now() - startedAt > WORKER_START_WAIT_MS);
+
+    await Bun.sleep(100);
+  }
+}
+
+function printVanishedWorker(logPath: string): number {
+  console.error(`atc daemon restart: the restart worker ended without a result; see ${logPath}`);
+
+  return 1;
+}
+
+function readLog(logPath: string): string {
+  try {
+    return readFileSync(logPath, 'utf8');
+  } catch {
+    return '';
+  }
+}
+
+function removeStaleRestartLogs(): void {
+  const cutoff = Date.now() - LOG_RETENTION_MS;
+
+  for (const entry of readdirSync(restartsDir)) {
+    if (!entry.endsWith('.log')) {
+      continue;
+    }
+
+    const path = join(restartsDir, entry);
+
+    try {
+      if (statSync(path).mtimeMs < cutoff) {
+        rmSync(path, { force: true });
+      }
+    } catch {}
+  }
+}

package/src/run-systemctl.ts

@@ -0,0 +1,33 @@
+interface SystemctlResult {
+  readonly code: number;
+  readonly stdout: string;
+  readonly stderr: string;
+}
+
+/**
+ * Runs `systemctl --user` with `args` and returns its exit code and output.
+ * A machine without systemctl reads as exit code 127.
+ */
+export async function runSystemctl(args: readonly string[]): Promise<SystemctlResult> {
+  try {
+    const proc = Bun.spawn(['systemctl', '--user', ...args], {
+      stdin: 'ignore',
+      stdout: 'pipe',
+      stderr: 'pipe',
+    });
+
+    const [stdout, stderr, code] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ]);
+
+    return { code, stdout: stdout.trim(), stderr: stderr.trim() };
+  } catch (error) {
+    return {
+      code: 127,
+      stdout: '',
+      stderr: error instanceof Error ? error.message : String(error),
+    };
+  }
+}

package/src/shared/build-atc-command.ts

@@ -0,0 +1,13 @@
+import { join } from 'node:path';
+import { isCompiledBinary } from './is-compiled-binary';
+
+/**
+ * The argv that runs this build of atc with `args`: the compiled binary is
+ * its own entry, and a source run is bun with the CLI entrypoint ahead of
+ * the arguments.
+ */
+export function buildATCCommand(args: readonly string[]): string[] {
+  return isCompiledBinary()
+    ? [process.execPath, ...args]
+    : [process.execPath, join(import.meta.dir, '..', 'cli.ts'), ...args];
+}

package/src/shared/config.ts

@@ -141,6 +141,11 @@ export const daemonPidFile = join(process.env['XDG_RUNTIME_DIR'] ?? stateDir, 'a
 // on, for a client whose environment computes other socket paths.
 export const daemonRecordFile = join(stateDir, 'daemon.json');
 
+// Where `atc daemon restart` keeps its run logs and the record of the last
+// finished restart, and the lock that admits one restart at a time.
+export const restartsDir = join(stateDir, 'restarts');
+export const restartLockFile = join(stateDir, 'daemon-restart.lock');
+
 // A user-written config.json's top-level keys. An absent or wrong-typed
 // field parses to undefined rather than failing the file, so a bad config
 // falls back to a default instead of refusing to start atc.

package/src/shared/find-pid-file-pid.ts

@@ -0,0 +1,15 @@
+import { readFileSync } from 'node:fs';
+
+/**
+ * The pid a daemon's pid file holds, or null when the file is missing or
+ * holds anything but a pid above 1.
+ */
+export function findPidFilePID(pidFile: string): number | null {
+  try {
+    const pid = Number(readFileSync(pidFile, 'utf8'));
+
+    return Number.isInteger(pid) && pid > 1 ? pid : null;
+  } catch {
+    return null;
+  }
+}

package/src/shared/get-build.ts

@@ -12,13 +12,14 @@ const SRC_ROOT = join(import.meta.dir, '..');
  * different checkouts of the same version still read as different builds,
  * which is what triggers the stale-daemon restart. Any source file counts:
  * a stamp taken from a single file misses every change that lands
- * elsewhere and leaves stale daemons in service.
+ * elsewhere and leaves stale daemons in service. The walk starts at this
+ * checkout's src/ root unless the caller passes another root.
  */
-export function getBuild(): string {
+export function getBuild(srcRoot: string = SRC_ROOT): string {
   let stamp = 0;
 
   try {
-    stamp = getNewestTSMtime(SRC_ROOT);
+    stamp = getNewestTSMtime(srcRoot);
   } catch {}
 
   if (stamp === 0) {

package/src/shared/is-process-alive.ts

@@ -0,0 +1,21 @@
+import { readFileSync } from 'node:fs';
+
+/**
+ * Whether a process with this pid is running. A zombie awaiting its parent's
+ * wait call has exited, so it reads as not running where `/proc` shows it.
+ */
+export function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+  } catch {
+    return false;
+  }
+
+  try {
+    const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
+
+    return !stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z');
+  } catch {
+    return true;
+  }
+}

package/src/shared/spawn-atc-detached.ts

@@ -0,0 +1,34 @@
+import { spawn as spawnChild } from 'node:child_process';
+import type { ChildProcess, StdioOptions } from 'node:child_process';
+import { buildATCCommand } from './build-atc-command';
+
+interface SpawnATCDetachedOptions {
+  readonly env?: Readonly<Record<string, string | undefined>>;
+  readonly cwd?: string;
+  readonly stdio?: StdioOptions;
+}
+
+/**
+ * Starts this build of atc with `args` in a session of its own, so it
+ * outlives the caller, and releases the handle so the caller can exit. The
+ * child's stdio is discarded unless `options` routes it elsewhere.
+ */
+export function spawnATCDetached(
+  args: readonly string[],
+
+  // oxlint-disable-next-line prefer-readonly-parameter-types -- the stdio option takes a mutable array
+  options: SpawnATCDetachedOptions = {},
+): ChildProcess {
+  const [bin = process.execPath, ...rest] = buildATCCommand(args);
+
+  const child = spawnChild(bin, rest, {
+    detached: true,
+    stdio: options.stdio ?? 'ignore',
+    ...(options.env === undefined ? {} : { env: options.env }),
+    ...(options.cwd === undefined ? {} : { cwd: options.cwd }),
+  });
+
+  child.unref();
+
+  return child;
+}

package/src/start-replacement-daemon.ts

@@ -0,0 +1,31 @@
+import type { DaemonProcess } from './daemon-process';
+import { spawnATCDetached } from './shared/spawn-atc-detached';
+
+interface ReplacementOverrides {
+  readonly listen: string | null;
+  readonly tokenFile: string | null;
+}
+
+/**
+ * Starts one daemon, detached, the way the one it replaces was started: its
+ * environment, its working directory, and its `--listen` and `--token-file`
+ * values, except where `overrides` sets them.
+ */
+export function startReplacementDaemon(
+  old: DaemonProcess,
+  overrides: ReplacementOverrides,
+): number | undefined {
+  const listen = overrides.listen ?? old.flags.listen;
+  const tokenFile = overrides.tokenFile ?? old.flags.tokenFile;
+
+  const child = spawnATCDetached(
+    [
+      'daemon',
+      ...(listen === null ? [] : ['--listen', listen]),
+      ...(tokenFile === null ? [] : ['--token-file', tokenFile]),
+    ],
+    { env: old.env, ...(old.cwd === null ? {} : { cwd: old.cwd }) },
+  );
+
+  return child.pid;
+}

package/src/stop-daemon-process.ts

@@ -0,0 +1,46 @@
+import { isProcessAlive } from './shared/is-process-alive';
+
+const EXIT_WAIT_MS = 10_000;
+
+/**
+ * Sends SIGTERM to the daemon and waits up to 10 s for it to exit, then
+ * sends SIGKILL when it is still running. Resolves `killed` when SIGKILL was
+ * needed and `exited` when SIGTERM was enough.
+ */
+export async function stopDaemonProcess(pid: number): Promise<'exited' | 'killed'> {
+  try {
+    process.kill(pid, 'SIGTERM');
+  } catch {
+    return 'exited';
+  }
+
+  const exited = await waitForExit(pid, EXIT_WAIT_MS);
+
+  if (exited) {
+    return 'exited';
+  }
+
+  try {
+    process.kill(pid, 'SIGKILL');
+  } catch {
+    return 'exited';
+  }
+
+  await waitForExit(pid, EXIT_WAIT_MS);
+
+  return 'killed';
+}
+
+async function waitForExit(pid: number, ms: number): Promise<boolean> {
+  const deadline = Date.now() + ms;
+
+  while (Date.now() < deadline) {
+    if (!isProcessAlive(pid)) {
+      return true;
+    }
+
+    await Bun.sleep(50);
+  }
+
+  return !isProcessAlive(pid);
+}

package/src/store/state-store.ts

@@ -539,14 +539,14 @@ export class StateStore {
     return row?.ts === null || row?.ts === undefined ? null : Date.parse(row.ts);
   }
 
-  async recordSpawnDir(cwd: string, grant: SpawnTarget): Promise<void> {
+  async recordSpawnDir(cwd: string, grant: SpawnTarget, at: number): Promise<void> {
     await this.db
       .insertInto('spawn_history')
       .values({
         cwd,
         target: grant.target,
         target_identity: grant.targetIdentity,
-        last_spawn: Date.now(),
+        last_spawn: at,
       })
       .onConflict((oc) =>
         oc

package/src/test-utils/can-bind-addresses.ts

@@ -0,0 +1,19 @@
+/**
+ * Whether this host can bind a socket to every given address, tried by
+ * listening on a kernel-chosen port of each and closing it at once. A host
+ * without an address, such as stock macOS for any loopback address past
+ * 127.0.0.1, refuses the bind with EADDRNOTAVAIL.
+ */
+export function canBindAddresses(hostnames: readonly string[]): boolean {
+  for (const hostname of hostnames) {
+    try {
+      const listener = Bun.listen({ hostname, port: 0, socket: { data() {} } });
+
+      listener.stop(true);
+    } catch {
+      return false;
+    }
+  }
+
+  return true;
+}

package/src/test-utils/create-migrated-state-db.ts

@@ -0,0 +1,38 @@
+import { copyFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { StateStore } from '../store/state-store';
+
+let template: Promise<string> | undefined;
+
+/**
+ * Creates a state database at the path that is already at the latest schema,
+ * by copying a database the store migrated once per test process. A store
+ * opened on the copy finds every migration applied and runs none, so a test
+ * that needs a ready store skips the migration ladder. Each copy is a file of
+ * its own; every copy holds the same daemon id, minted when the template was
+ * migrated. The template sits in the test home, which the test script removes
+ * on exit.
+ */
+export async function createMigratedStateDB(dbPath: string): Promise<void> {
+  template ??= createTemplateStateDB();
+
+  const templatePath = await template;
+
+  copyFileSync(templatePath, dbPath);
+}
+
+async function createTemplateStateDB(): Promise<string> {
+  const home = process.env['ATC_TEST_HOME'];
+
+  if (home === undefined) {
+    throw new Error('ATC_TEST_HOME is unset; run the tests through `bun run test`');
+  }
+
+  const path = join(home, 'migrated-state.db');
+
+  const store = await StateStore.open(path);
+
+  await store.stop();
+
+  return path;
+}

package/src/test-utils/env-originals.ts

@@ -0,0 +1,6 @@
+/**
+ * Holds each environment variable a test has overridden, keyed by name, with
+ * the value it held before its first override in that test; `undefined`
+ * records a variable that was unset.
+ */
+export const envOriginals = new Map<string, string | undefined>();