@zgeoff/atc 3.2.0 → 3.3.0

package/README.md

@@ -98,6 +98,12 @@ fleet when you are ready. The bar shows `⟳ restarting daemon` until the restar
 running daemon speaks another protocol version, `atc` asks before it restarts the daemon, since the
 restart ends every session the daemon hosts.
 
+`atc daemon restart` does the same from a script or from inside a session: it stops the daemon,
+starts one in its place, restores the fleet, and exits 0 only when every stored row came back.
+`atc daemon restart --dry-run` prints what the restart would stop and which sessions are mid-turn.
+[Daemon architecture](./docs/architecture/daemon.md#restarting-the-daemon) covers the handoff and
+the exit codes.
+
 atc runs inside zellij or tmux. Give the pane locked mode so the leader key reaches atc.
 
 ## Grok and Codex

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zgeoff/atc",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "description": "Terminal control tower for coding-agent sessions",
   "homepage": "https://github.com/zgeoff/atc#readme",
   "bugs": "https://github.com/zgeoff/atc/issues",
@@ -28,8 +28,9 @@
     "deadcode": "knip",
     "lint": "oxlint --type-aware --type-check --report-unused-disable-directives-severity error",
     "lint:fix": "oxlint --type-aware --type-check --report-unused-disable-directives-severity error --fix",
-    "test": "bash scripts/with-test-home.sh bun test",
-    "test:binary": "bash scripts/with-test-home.sh bun test --timeout 20000 test/daemon-e2e.test.ts",
+    "test": "bash scripts/with-test-home.sh bun test --parallel --timings=bun-test-timings.json",
+    "test:timings": "bash scripts/with-test-home.sh bun test --parallel --timings=bun-test-timings.json --update-timings",
+    "test:binary": "bash scripts/with-test-home.sh bun test --timeout 20000 e2e/daemon-e2e.test.ts",
     "test:gateway-binary": "bash scripts/with-test-home.sh bun test --timeout 20000 src/gateway.test.ts",
     "test:atc-bridge": "bash scripts/with-test-home.sh bash scripts/test-atc-bridge.sh",
     "test:isolation": "bash scripts/check-test-isolation.sh",

package/src/build-own-daemon-process.ts

@@ -0,0 +1,16 @@
+import { collectRestartEnv } from './collect-restart-env';
+import type { DaemonProcess } from './daemon-process';
+
+/**
+ * The start a daemon gets when no running one supplies it: this process's
+ * environment without the session variables, no working directory, and no
+ * listener flags.
+ */
+export function buildOwnDaemonProcess(): DaemonProcess {
+  return {
+    env: collectRestartEnv(process.env),
+    cwd: null,
+    flags: { listen: null, tokenFile: null },
+    fromProc: false,
+  };
+}

package/src/build-restart-failure.ts

@@ -0,0 +1,24 @@
+import type { RestartResult } from './parse-restart-result';
+
+/**
+ * The result of a restart that stopped with `error` before it could count
+ * the fleet. `interrupted` lists the sessions the stop already ended.
+ */
+export function buildRestartFailure(
+  runID: string,
+  error: string,
+  interrupted: RestartResult['interrupted'] = [],
+): RestartResult {
+  return {
+    runID,
+    code: 1,
+    pid: null,
+    build: null,
+    listenPort: null,
+    restored: 0,
+    total: 0,
+    failed: [],
+    interrupted,
+    error,
+  };
+}

package/src/cli.ts

@@ -1,5 +1,6 @@
 // atc CLI entry: no subcommand opens the client TUI; `hook-report` and
 // `statusline` are the commands injected into wrangled sessions.
+import { resolve } from 'node:path';
 import { defineCommand, runMain } from 'citty';
 import pkg from '../package.json';
 import { collectRedirectURIs } from './collect-redirect-uris';
@@ -252,6 +253,64 @@ const main = defineCommand({
                 await id.runDaemonID(getBuild());
               },
             }),
+          restart: () =>
+            defineCommand({
+              meta: {
+                name: 'restart',
+                description: 'Restart the running daemon and restore its fleet',
+              },
+              args: {
+                ...DAEMON_ARGS,
+                timeout: {
+                  type: 'string',
+                  description: 'Seconds to wait for the restored fleet to come alive',
+                },
+                'dry-run': {
+                  type: 'boolean',
+                  default: false,
+                  description: 'Print what the restart would do and stop',
+                },
+              },
+              async run(ctx) {
+                const options = parseRestartArgs(ctx.args);
+
+                const restart = await import('./run-daemon-restart');
+
+                const code = await restart.runDaemonRestart({
+                  ...options,
+                  dryRun: ctx.args['dry-run'],
+                });
+
+                process.exit(code);
+              },
+            }),
+          'restart-worker': () =>
+            defineCommand({
+              meta: {
+                name: 'restart-worker',
+                description: 'Run one daemon restart for atc daemon restart',
+                hidden: true,
+              },
+              args: {
+                ...DAEMON_ARGS,
+                runID: { type: 'positional', required: true, description: 'The run id' },
+                session: { type: 'string', description: 'The session that asked for the restart' },
+                timeout: { type: 'string', description: 'Seconds to wait for the restored fleet' },
+              },
+              async run(ctx) {
+                const options = parseRestartArgs(ctx.args);
+
+                const worker = await import('./run-daemon-restart-worker');
+
+                const code = await worker.runDaemonRestartWorker({
+                  ...options,
+                  runID: ctx.args.runID,
+                  callerSession: ctx.args.session ?? null,
+                });
+
+                process.exit(code);
+              },
+            }),
         },
       }),
     events: () =>
@@ -379,6 +438,43 @@ const main = defineCommand({
   },
 });
 
+interface RestartArgs {
+  readonly listen?: string | undefined;
+  readonly 'token-file'?: string | undefined;
+  readonly timeout?: string | undefined;
+}
+
+interface RestartFlags {
+  readonly listen: string | null;
+  readonly tokenFile: string | null;
+  readonly timeoutSeconds: number | null;
+}
+
+// Reads the flags the restart commands share, and stops the command on one
+// the replacement daemon would refuse after the old one is already gone.
+function parseRestartArgs(args: RestartArgs): RestartFlags {
+  const listen = args.listen ?? null;
+  const parsedListen = listen === null ? null : parseListenAddress(listen);
+
+  if (parsedListen !== null && !parsedListen.ok) {
+    console.error(`atc daemon restart: ${parsedListen.message}`);
+    process.exit(1);
+  }
+
+  const timeout = args.timeout === undefined ? null : Number(args.timeout);
+
+  if (timeout !== null && (!Number.isFinite(timeout) || timeout <= 0)) {
+    console.error('atc daemon restart: --timeout takes a number of seconds above 0');
+    process.exit(1);
+  }
+
+  // An explicit token file is read where the caller runs, not in the old
+  // daemon's directory, so a relative path is fixed here before the handoff.
+  const tokenFile = args['token-file'] === undefined ? null : resolve(args['token-file']);
+
+  return { listen, tokenFile, timeoutSeconds: timeout };
+}
+
 // Runs the daemon in the foreground until SIGTERM, with a TCP listener when
 // --listen and --token-file are given.
 async function runDaemon(listenArg: string | null, tokenFile: string | null): Promise<void> {

package/src/client/boot-daemon.ts

@@ -1,7 +1,4 @@
-import { spawn as spawnChild } from 'node:child_process';
 import type { ChildProcess } from 'node:child_process';
-import { readFileSync } from 'node:fs';
-import { join } from 'node:path';
 import { DaemonError } from '../protocol/daemon-error';
 import type { DaemonFeature } from '../protocol/daemon-features';
 import { parseDaemonFeatures } from '../protocol/parse-daemon-features';
@@ -9,9 +6,11 @@ import { PROTOCOL_V } from '../protocol/protocol';
 import type { AgentID } from '../shared/agent-id';
 import { daemonPidFile, daemonRecordFile, daemonSocketPath } from '../shared/config';
 import { findDaemonRecord } from '../shared/find-daemon-record';
+import { findPidFilePID } from '../shared/find-pid-file-pid';
 import { getBuild } from '../shared/get-build';
-import { isCompiledBinary } from '../shared/is-compiled-binary';
+import { isProcessAlive } from '../shared/is-process-alive';
 import { makeSingleFlight } from '../shared/make-single-flight';
+import { spawnATCDetached } from '../shared/spawn-atc-detached';
 import { toAgentID } from '../shared/to-agent-id';
 import { DaemonClient } from './daemon-client';
 import { formatProtocolMismatch } from './format-protocol-mismatch';
@@ -275,23 +274,13 @@ function formatWaitFailure(timeoutMs: number): string {
   );
 }
 
-function isProcessAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 // The pid of the daemon behind the socket that refused the handshake.
 function findDaemonPID(socketPath: string): number | null {
   return pickStaleDaemonPID({
     socketPath,
     record: findDaemonRecord(daemonRecordFile),
     pidFileSocketPath: daemonSocketPath,
-    pidFilePID: findPidFilePID(),
+    pidFilePID: findPidFilePID(daemonPidFile),
   });
 }
 
@@ -315,22 +304,6 @@ async function stopDaemon(pid: number): Promise<void> {
   }
 }
 
-// The pid file beside the sockets this environment computes.
-function findPidFilePID(): number | null {
-  try {
-    const pid = Number(readFileSync(daemonPidFile, 'utf8'));
-
-    return Number.isInteger(pid) && pid > 1 ? pid : null;
-  } catch {
-    return null;
-  }
-}
-
 function spawnDaemonDetached(): ChildProcess {
-  const args = isCompiledBinary() ? ['daemon'] : [join(import.meta.dir, '..', 'cli.ts'), 'daemon'];
-  const child = spawnChild(process.execPath, args, { detached: true, stdio: 'ignore' });
-
-  child.unref();
-
-  return child;
+  return spawnATCDetached(['daemon']);
 }

package/src/collect-restart-env.ts

@@ -0,0 +1,18 @@
+/**
+ * Copies an environment without the variables that name the session or
+ * socket of whoever runs the restart, so the daemon it starts never mistakes
+ * itself for a hosted session. Entries with no value drop out.
+ */
+export function collectRestartEnv(
+  env: Readonly<Record<string, string | undefined>>,
+): Record<string, string> {
+  const copy: Record<string, string> = {};
+
+  for (const [key, value] of Object.entries(env)) {
+    if (value !== undefined && key !== 'ATC_SESSION_ID' && key !== 'ATC_SOCKET') {
+      copy[key] = value;
+    }
+  }
+
+  return copy;
+}

package/src/collect-restart-plan.ts

@@ -0,0 +1,157 @@
+import { DaemonClient } from './client/daemon-client';
+import { pickStaleDaemonPID } from './client/pick-stale-daemon-pid';
+import { findRestartUnit } from './find-restart-unit';
+import { isDaemonProcess } from './is-daemon-process';
+import { DaemonError } from './protocol/daemon-error';
+import { PROTOCOL_V } from './protocol/protocol';
+import type { DaemonAnswer, PlanSession, ReplacementPlan, RestartPlan } from './restart-plan';
+import { runSystemctl } from './run-systemctl';
+import { buildATCCommand } from './shared/build-atc-command';
+import { daemonPidFile, daemonRecordFile, daemonSocketPath, stateDir } from './shared/config';
+import { findDaemonRecord } from './shared/find-daemon-record';
+import { findPidFilePID } from './shared/find-pid-file-pid';
+import { getBuild } from './shared/get-build';
+import { isProcessAlive } from './shared/is-process-alive';
+import { isRecord } from './shared/report';
+
+// How long the handshake and the session list may take together.
+const PROBE_DEADLINE_MS = 3000;
+
+interface Probe {
+  readonly socketPath: string;
+  readonly answer: DaemonAnswer;
+  readonly sessions: readonly PlanSession[] | null;
+}
+
+/**
+ * Finds the daemon a restart would stop and the replacement it would start,
+ * without changing either. The daemon answers on the socket this
+ * environment computes or the one its record in the state directory holds,
+ * and a handshake that its protocol refuses still identifies it.
+ */
+export async function collectRestartPlan(callerSession: string | null): Promise<RestartPlan> {
+  const record = findDaemonRecord(daemonRecordFile);
+
+  const paths = new Set([daemonSocketPath, ...(record === null ? [] : [record.socketPath])]);
+
+  let probe: Probe | null = null;
+
+  for (const path of paths) {
+    probe = await tryProbe(path);
+
+    if (probe !== null) {
+      break;
+    }
+  }
+
+  const pid =
+    probe === null
+      ? pickLivePID(record?.pid ?? null, findPidFilePID(daemonPidFile))
+      : (pickStaleDaemonPID({
+          socketPath: probe.socketPath,
+          record,
+          pidFileSocketPath: daemonSocketPath,
+          pidFilePID: findPidFilePID(daemonPidFile),
+        }) ?? pickLivePID(record?.pid ?? null, null));
+
+  return {
+    pid,
+    socketPath: probe?.socketPath ?? null,
+    answer: probe?.answer ?? null,
+    sessions: probe?.sessions ?? null,
+    callerSession,
+    replacement: await planReplacement(pid),
+  };
+}
+
+function pickLivePID(...candidates: readonly (number | null)[]): number | null {
+  return (
+    candidates.find(
+      (pid) => pid !== null && isProcessAlive(pid) && isDaemonProcess(pid, stateDir),
+    ) ?? null
+  );
+}
+
+async function planReplacement(pid: number | null): Promise<ReplacementPlan> {
+  const unit = pid === null ? null : await findRestartUnit(pid);
+
+  if (unit === null) {
+    return { kind: 'plain', build: getBuild(), command: buildATCCommand(['daemon']).join(' ') };
+  }
+
+  const shown = await runSystemctl(['show', '-p', 'ExecStart', '--value', unit]);
+
+  const execPath = /path=(?<path>[^\s;]+)/.exec(shown.stdout)?.groups?.['path'];
+
+  return {
+    kind: 'unit',
+    unit,
+    execStart: execPath ?? (shown.code === 0 && shown.stdout !== '' ? shown.stdout : null),
+  };
+}
+
+async function tryProbe(socketPath: string): Promise<Probe | null> {
+  let client: DaemonClient;
+
+  try {
+    client = await DaemonClient.open(socketPath);
+  } catch {
+    return null;
+  }
+
+  // A daemon that accepts the socket and never answers must not hang the
+  // restart: closing the client rejects every request still waiting.
+  const expiry = setTimeout(() => {
+    client.stop();
+  }, PROBE_DEADLINE_MS);
+
+  try {
+    const hello = await client.sendHello(getBuild());
+
+    const build = hello['daemon'];
+
+    return {
+      socketPath,
+      answer: {
+        kind: 'ok',
+        build: typeof build === 'string' ? build : 'unknown',
+        protocol: PROTOCOL_V,
+      },
+      sessions: await tryListSessions(client),
+    };
+  } catch (error) {
+    if (error instanceof DaemonError && error.code === 'protocol_mismatch') {
+      return { socketPath, answer: { kind: 'refused', message: error.message }, sessions: null };
+    }
+
+    return null;
+  } finally {
+    clearTimeout(expiry);
+
+    client.stop();
+  }
+}
+
+async function tryListSessions(
+  client: Pick<DaemonClient, 'sendRequest'>,
+): Promise<PlanSession[] | null> {
+  try {
+    const listed = await client.sendRequest('session.list');
+
+    const sessions = listed['sessions'];
+
+    if (!Array.isArray(sessions)) {
+      return null;
+    }
+
+    return sessions
+      .filter((entry) => isRecord(entry))
+      .map((entry) => ({
+        id: String(entry['id']),
+        name: String(entry['name']),
+        state: String(entry['state']),
+      }));
+  } catch {
+    return null;
+  }
+}

package/src/daemon/daemon.ts

@@ -12,11 +12,13 @@ import type { EventMsg } from '../protocol/protocol';
 import type { SpawnWorkspaceSource } from '../protocol/request-param-schemas';
 import type { SessionState } from '../protocol/session-state';
 import type { AgentID } from '../shared/agent-id';
+import { claimDaemonLock } from '../shared/claim-daemon-lock';
 import type { HooksConfig } from '../shared/collect-hooks';
 import type { TargetConfigError } from '../shared/collect-targets';
 import type { InvalidGitTransports } from '../shared/collect-workspaces-config';
 import { DEFAULT_GIT_TRANSPORTS } from '../shared/default-git-transports';
 import { findDaemonRecord } from '../shared/find-daemon-record';
+import { loadListenerTokens } from '../shared/load-listener-tokens';
 import type { MessageID } from '../shared/message-id';
 import { isRecord } from '../shared/report';
 import { resolveHomeDir } from '../shared/resolve-home-dir';
@@ -50,7 +52,6 @@ import { buildSessionReportEvent } from './build-session-report-event';
 import { buildTargetAccess } from './build-target-access';
 import { buildTargetForbiddenError } from './build-target-forbidden-error';
 import { buildTargetList } from './build-target-list';
-import { claimDaemonLock } from './claim-daemon-lock';
 import { createNonBlockingLog } from './create-non-blocking-log';
 import type { NonBlockingLog } from './create-non-blocking-log';
 import { DaemonConnection } from './daemon-connection';
@@ -70,7 +71,6 @@ import { IdempotencyLedger } from './idempotency-ledger';
 import { isAllowedListenHost } from './is-allowed-listen-host';
 import { isOwnHookEvent } from './is-own-hook-event';
 import { isTreeInReach } from './is-tree-in-reach';
-import { loadListenerTokens } from './load-listener-tokens';
 import { loadTranscriptPage } from './load-transcript-page';
 import { makeHookRunner } from './make-hook-runner';
 import type { HookScope } from './make-hook-runner';
@@ -1112,7 +1112,11 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       runtime.pendingLastUsed = true;
     }
 
-    void store.recordSpawnDir(s.cwd, { target: s.target, targetIdentity: s.targetIdentity });
+    void store.recordSpawnDir(
+      s.cwd,
+      { target: s.target, targetIdentity: s.targetIdentity },
+      Date.now(),
+    );
 
     return getDescriptor(mgr, s.id);
   };

package/src/daemon-process.ts

@@ -0,0 +1,12 @@
+import type { DaemonFlags } from './parse-daemon-flags';
+
+// What a daemon was started with.
+export interface DaemonProcess {
+  readonly env: Readonly<Record<string, string>>;
+  readonly cwd: string | null;
+  readonly flags: DaemonFlags;
+
+  // False when `/proc` could not be read, in which case `env` is this
+  // process's own.
+  readonly fromProc: boolean;
+}

package/src/find-restart-unit.ts

@@ -0,0 +1,45 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { parseSystemdUnit } from './parse-systemd-unit';
+import { runSystemctl } from './run-systemctl';
+
+/**
+ * The user service whose main process is the daemon with this pid, or null
+ * when the daemon is not one.
+ *
+ * Two facts must hold. The daemon's cgroup sits in a `<name>.service` below
+ * the user manager, and the manager reports that unit's `MainPID` as this
+ * pid. The cgroup alone proves nothing: every process started from a
+ * session inside the unit, such as a daemon a test starts, shares the
+ * unit's cgroup without being the process the unit runs. `ATC_PROC_ROOT`
+ * moves the cgroup read from `/proc` to another directory, so a test can
+ * place a daemon in a unit without a user manager.
+ */
+export async function findRestartUnit(pid: number): Promise<string | null> {
+  const uid = process.getuid?.();
+
+  if (uid === undefined) {
+    return null;
+  }
+
+  let cgroup: string;
+
+  try {
+    cgroup = readFileSync(
+      join(process.env['ATC_PROC_ROOT'] ?? '/proc', String(pid), 'cgroup'),
+      'utf8',
+    );
+  } catch {
+    return null;
+  }
+
+  const unit = parseSystemdUnit(cgroup, uid);
+
+  if (unit === null) {
+    return null;
+  }
+
+  const shown = await runSystemctl(['show', '-p', 'MainPID', '--value', unit]);
+
+  return shown.code === 0 && shown.stdout === String(pid) ? unit : null;
+}

package/src/format-restart-preflight.ts

@@ -0,0 +1,74 @@
+import { parseMismatchBuild } from './parse-mismatch-build';
+import type { RestartPlan } from './restart-plan';
+
+const INTERRUPTION_POLICY =
+  'Stopping the daemon ends every agent process it hosts. A session that is mid-turn loses that turn; the restore resumes each session from its transcript, and the interrupted turn does not continue.';
+
+/**
+ * Renders what a restart is about to do: the daemon it stops, the sessions
+ * that are mid-turn, the replacement it starts, and the interruption
+ * policy.
+ */
+export function formatRestartPreflight(plan: RestartPlan): string[] {
+  return [...formatTarget(plan), ...formatReplacement(plan), INTERRUPTION_POLICY];
+}
+
+function formatTarget(plan: RestartPlan): string[] {
+  const pid = plan.pid === null ? 'pid unknown' : `pid ${plan.pid}`;
+
+  if (plan.answer === null) {
+    return [
+      plan.pid === null
+        ? 'no daemon answers and no live pid is recorded; the restart only starts one and restores the fleet'
+        : `the daemon (${pid}) does not answer; the restart stops it, starts one, and restores the fleet`,
+    ];
+  }
+
+  if (plan.answer.kind === 'refused') {
+    const parsed = parseMismatchBuild(plan.answer.message);
+
+    return [
+      `daemon: ${pid} refused this build's handshake: ${plan.answer.message}`,
+      ...(parsed === null
+        ? []
+        : [`daemon build ${parsed.build} speaks protocol v${parsed.protocol}`]),
+      'The session states cannot be read across the protocol mismatch, so the sessions that are mid-turn are unknown.',
+    ];
+  }
+
+  return [
+    `daemon: ${pid}, build ${plan.answer.build}, protocol v${plan.answer.protocol}`,
+    ...formatSessions(plan),
+  ];
+}
+
+function formatSessions(plan: RestartPlan): string[] {
+  if (plan.sessions === null) {
+    return ['The session list could not be read, so the sessions that are mid-turn are unknown.'];
+  }
+
+  const running = plan.sessions.filter((session) => session.state === 'running');
+
+  if (running.length === 0) {
+    return ['no session is mid-turn'];
+  }
+
+  return [
+    `${running.length} session${running.length === 1 ? ' is' : 's are'} mid-turn:`,
+    ...running.map(
+      (session) =>
+        `  ${session.name} (${session.id})${session.id === plan.callerSession ? ' (this session)' : ''}`,
+    ),
+  ];
+}
+
+function formatReplacement(plan: RestartPlan): string[] {
+  if (plan.replacement.kind === 'unit') {
+    return [
+      `replacement: systemd unit ${plan.replacement.unit}${plan.replacement.execStart === null ? '' : `, ExecStart ${plan.replacement.execStart}`}`,
+      'The unit decides the build the replacement runs.',
+    ];
+  }
+
+  return [`replacement: build ${plan.replacement.build}, started with ${plan.replacement.command}`];
+}

package/src/format-restart-report.ts

@@ -0,0 +1,45 @@
+import type { RestartResult } from './parse-restart-result';
+
+/**
+ * Renders a finished restart as the lines the run prints: the daemon now
+ * serving, how much of the fleet came back, each row that did not, and the
+ * sessions the restart interrupted.
+ */
+export function formatRestartReport(result: RestartResult): string[] {
+  const lines: string[] = [];
+
+  if (result.error !== null) {
+    lines.push(`restart failed: ${result.error}`);
+  }
+
+  if (result.pid !== null) {
+    lines.push(`daemon pid ${result.pid}, build ${result.build ?? 'unknown'}`);
+  }
+
+  if (result.listenPort !== null) {
+    lines.push(`listening on port ${result.listenPort}`);
+  }
+
+  if (result.error === null) {
+    lines.push(`restored ${result.restored} of ${result.total}`);
+  }
+
+  for (const row of result.failed) {
+    lines.push(`failed: ${row.name} (${row.id}): ${row.reason}`);
+  }
+
+  if (result.interrupted.length > 0) {
+    lines.push(`interrupted ${result.interrupted.length} mid-turn:`);
+
+    for (const row of result.interrupted) {
+      lines.push(`  ${row.name} (${row.id})`);
+    }
+  }
+
+  const verdict =
+    result.code === 0 ? 'restart succeeded' : `restart failed with exit code ${result.code}`;
+
+  lines.push(verdict);
+
+  return lines;
+}