@zgeoff/atc 3.1.1 → 3.3.0

package/README.md

@@ -98,6 +98,12 @@ fleet when you are ready. The bar shows `⟳ restarting daemon` until the restar
 running daemon speaks another protocol version, `atc` asks before it restarts the daemon, since the
 restart ends every session the daemon hosts.
 
+`atc daemon restart` does the same from a script or from inside a session: it stops the daemon,
+starts one in its place, restores the fleet, and exits 0 only when every stored row came back.
+`atc daemon restart --dry-run` prints what the restart would stop and which sessions are mid-turn.
+[Daemon architecture](./docs/architecture/daemon.md#restarting-the-daemon) covers the handoff and
+the exit codes.
+
 atc runs inside zellij or tmux. Give the pane locked mode so the leader key reaches atc.
 
 ## Grok and Codex

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zgeoff/atc",
-  "version": "3.1.1",
+  "version": "3.3.0",
   "description": "Terminal control tower for coding-agent sessions",
   "homepage": "https://github.com/zgeoff/atc#readme",
   "bugs": "https://github.com/zgeoff/atc/issues",
@@ -28,8 +28,9 @@
     "deadcode": "knip",
     "lint": "oxlint --type-aware --type-check --report-unused-disable-directives-severity error",
     "lint:fix": "oxlint --type-aware --type-check --report-unused-disable-directives-severity error --fix",
-    "test": "bash scripts/with-test-home.sh bun test",
-    "test:binary": "bash scripts/with-test-home.sh bun test --timeout 20000 test/daemon-e2e.test.ts",
+    "test": "bash scripts/with-test-home.sh bun test --parallel --timings=bun-test-timings.json",
+    "test:timings": "bash scripts/with-test-home.sh bun test --parallel --timings=bun-test-timings.json --update-timings",
+    "test:binary": "bash scripts/with-test-home.sh bun test --timeout 20000 e2e/daemon-e2e.test.ts",
     "test:gateway-binary": "bash scripts/with-test-home.sh bun test --timeout 20000 src/gateway.test.ts",
     "test:atc-bridge": "bash scripts/with-test-home.sh bash scripts/test-atc-bridge.sh",
     "test:isolation": "bash scripts/check-test-isolation.sh",

package/src/build-own-daemon-process.ts

@@ -0,0 +1,16 @@
+import { collectRestartEnv } from './collect-restart-env';
+import type { DaemonProcess } from './daemon-process';
+
+/**
+ * The start a daemon gets when no running one supplies it: this process's
+ * environment without the session variables, no working directory, and no
+ * listener flags.
+ */
+export function buildOwnDaemonProcess(): DaemonProcess {
+  return {
+    env: collectRestartEnv(process.env),
+    cwd: null,
+    flags: { listen: null, tokenFile: null },
+    fromProc: false,
+  };
+}

package/src/build-restart-failure.ts

@@ -0,0 +1,24 @@
+import type { RestartResult } from './parse-restart-result';
+
+/**
+ * The result of a restart that stopped with `error` before it could count
+ * the fleet. `interrupted` lists the sessions the stop already ended.
+ */
+export function buildRestartFailure(
+  runID: string,
+  error: string,
+  interrupted: RestartResult['interrupted'] = [],
+): RestartResult {
+  return {
+    runID,
+    code: 1,
+    pid: null,
+    build: null,
+    listenPort: null,
+    restored: 0,
+    total: 0,
+    failed: [],
+    interrupted,
+    error,
+  };
+}

package/src/cli.ts

@@ -1,5 +1,6 @@
 // atc CLI entry: no subcommand opens the client TUI; `hook-report` and
 // `statusline` are the commands injected into wrangled sessions.
+import { resolve } from 'node:path';
 import { defineCommand, runMain } from 'citty';
 import pkg from '../package.json';
 import { collectRedirectURIs } from './collect-redirect-uris';
@@ -252,6 +253,64 @@ const main = defineCommand({
                 await id.runDaemonID(getBuild());
               },
             }),
+          restart: () =>
+            defineCommand({
+              meta: {
+                name: 'restart',
+                description: 'Restart the running daemon and restore its fleet',
+              },
+              args: {
+                ...DAEMON_ARGS,
+                timeout: {
+                  type: 'string',
+                  description: 'Seconds to wait for the restored fleet to come alive',
+                },
+                'dry-run': {
+                  type: 'boolean',
+                  default: false,
+                  description: 'Print what the restart would do and stop',
+                },
+              },
+              async run(ctx) {
+                const options = parseRestartArgs(ctx.args);
+
+                const restart = await import('./run-daemon-restart');
+
+                const code = await restart.runDaemonRestart({
+                  ...options,
+                  dryRun: ctx.args['dry-run'],
+                });
+
+                process.exit(code);
+              },
+            }),
+          'restart-worker': () =>
+            defineCommand({
+              meta: {
+                name: 'restart-worker',
+                description: 'Run one daemon restart for atc daemon restart',
+                hidden: true,
+              },
+              args: {
+                ...DAEMON_ARGS,
+                runID: { type: 'positional', required: true, description: 'The run id' },
+                session: { type: 'string', description: 'The session that asked for the restart' },
+                timeout: { type: 'string', description: 'Seconds to wait for the restored fleet' },
+              },
+              async run(ctx) {
+                const options = parseRestartArgs(ctx.args);
+
+                const worker = await import('./run-daemon-restart-worker');
+
+                const code = await worker.runDaemonRestartWorker({
+                  ...options,
+                  runID: ctx.args.runID,
+                  callerSession: ctx.args.session ?? null,
+                });
+
+                process.exit(code);
+              },
+            }),
         },
       }),
     events: () =>
@@ -379,6 +438,43 @@ const main = defineCommand({
   },
 });
 
+interface RestartArgs {
+  readonly listen?: string | undefined;
+  readonly 'token-file'?: string | undefined;
+  readonly timeout?: string | undefined;
+}
+
+interface RestartFlags {
+  readonly listen: string | null;
+  readonly tokenFile: string | null;
+  readonly timeoutSeconds: number | null;
+}
+
+// Reads the flags the restart commands share, and stops the command on one
+// the replacement daemon would refuse after the old one is already gone.
+function parseRestartArgs(args: RestartArgs): RestartFlags {
+  const listen = args.listen ?? null;
+  const parsedListen = listen === null ? null : parseListenAddress(listen);
+
+  if (parsedListen !== null && !parsedListen.ok) {
+    console.error(`atc daemon restart: ${parsedListen.message}`);
+    process.exit(1);
+  }
+
+  const timeout = args.timeout === undefined ? null : Number(args.timeout);
+
+  if (timeout !== null && (!Number.isFinite(timeout) || timeout <= 0)) {
+    console.error('atc daemon restart: --timeout takes a number of seconds above 0');
+    process.exit(1);
+  }
+
+  // An explicit token file is read where the caller runs, not in the old
+  // daemon's directory, so a relative path is fixed here before the handoff.
+  const tokenFile = args['token-file'] === undefined ? null : resolve(args['token-file']);
+
+  return { listen, tokenFile, timeoutSeconds: timeout };
+}
+
 // Runs the daemon in the foreground until SIGTERM, with a TCP listener when
 // --listen and --token-file are given.
 async function runDaemon(listenArg: string | null, tokenFile: string | null): Promise<void> {

package/src/client/boot-daemon.ts

@@ -1,7 +1,4 @@
-import { spawn as spawnChild } from 'node:child_process';
 import type { ChildProcess } from 'node:child_process';
-import { readFileSync } from 'node:fs';
-import { join } from 'node:path';
 import { DaemonError } from '../protocol/daemon-error';
 import type { DaemonFeature } from '../protocol/daemon-features';
 import { parseDaemonFeatures } from '../protocol/parse-daemon-features';
@@ -9,9 +6,11 @@ import { PROTOCOL_V } from '../protocol/protocol';
 import type { AgentID } from '../shared/agent-id';
 import { daemonPidFile, daemonRecordFile, daemonSocketPath } from '../shared/config';
 import { findDaemonRecord } from '../shared/find-daemon-record';
+import { findPidFilePID } from '../shared/find-pid-file-pid';
 import { getBuild } from '../shared/get-build';
-import { isCompiledBinary } from '../shared/is-compiled-binary';
+import { isProcessAlive } from '../shared/is-process-alive';
 import { makeSingleFlight } from '../shared/make-single-flight';
+import { spawnATCDetached } from '../shared/spawn-atc-detached';
 import { toAgentID } from '../shared/to-agent-id';
 import { DaemonClient } from './daemon-client';
 import { formatProtocolMismatch } from './format-protocol-mismatch';
@@ -275,23 +274,13 @@ function formatWaitFailure(timeoutMs: number): string {
   );
 }
 
-function isProcessAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 // The pid of the daemon behind the socket that refused the handshake.
 function findDaemonPID(socketPath: string): number | null {
   return pickStaleDaemonPID({
     socketPath,
     record: findDaemonRecord(daemonRecordFile),
     pidFileSocketPath: daemonSocketPath,
-    pidFilePID: findPidFilePID(),
+    pidFilePID: findPidFilePID(daemonPidFile),
   });
 }
 
@@ -315,22 +304,6 @@ async function stopDaemon(pid: number): Promise<void> {
   }
 }
 
-// The pid file beside the sockets this environment computes.
-function findPidFilePID(): number | null {
-  try {
-    const pid = Number(readFileSync(daemonPidFile, 'utf8'));
-
-    return Number.isInteger(pid) && pid > 1 ? pid : null;
-  } catch {
-    return null;
-  }
-}
-
 function spawnDaemonDetached(): ChildProcess {
-  const args = isCompiledBinary() ? ['daemon'] : [join(import.meta.dir, '..', 'cli.ts'), 'daemon'];
-  const child = spawnChild(process.execPath, args, { detached: true, stdio: 'ignore' });
-
-  child.unref();
-
-  return child;
+  return spawnATCDetached(['daemon']);
 }

package/src/collect-restart-env.ts

@@ -0,0 +1,18 @@
+/**
+ * Copies an environment without the variables that name the session or
+ * socket of whoever runs the restart, so the daemon it starts never mistakes
+ * itself for a hosted session. Entries with no value drop out.
+ */
+export function collectRestartEnv(
+  env: Readonly<Record<string, string | undefined>>,
+): Record<string, string> {
+  const copy: Record<string, string> = {};
+
+  for (const [key, value] of Object.entries(env)) {
+    if (value !== undefined && key !== 'ATC_SESSION_ID' && key !== 'ATC_SOCKET') {
+      copy[key] = value;
+    }
+  }
+
+  return copy;
+}

package/src/collect-restart-plan.ts

@@ -0,0 +1,157 @@
+import { DaemonClient } from './client/daemon-client';
+import { pickStaleDaemonPID } from './client/pick-stale-daemon-pid';
+import { findRestartUnit } from './find-restart-unit';
+import { isDaemonProcess } from './is-daemon-process';
+import { DaemonError } from './protocol/daemon-error';
+import { PROTOCOL_V } from './protocol/protocol';
+import type { DaemonAnswer, PlanSession, ReplacementPlan, RestartPlan } from './restart-plan';
+import { runSystemctl } from './run-systemctl';
+import { buildATCCommand } from './shared/build-atc-command';
+import { daemonPidFile, daemonRecordFile, daemonSocketPath, stateDir } from './shared/config';
+import { findDaemonRecord } from './shared/find-daemon-record';
+import { findPidFilePID } from './shared/find-pid-file-pid';
+import { getBuild } from './shared/get-build';
+import { isProcessAlive } from './shared/is-process-alive';
+import { isRecord } from './shared/report';
+
+// How long the handshake and the session list may take together.
+const PROBE_DEADLINE_MS = 3000;
+
+interface Probe {
+  readonly socketPath: string;
+  readonly answer: DaemonAnswer;
+  readonly sessions: readonly PlanSession[] | null;
+}
+
+/**
+ * Finds the daemon a restart would stop and the replacement it would start,
+ * without changing either. The daemon answers on the socket this
+ * environment computes or the one its record in the state directory holds,
+ * and a handshake that its protocol refuses still identifies it.
+ */
+export async function collectRestartPlan(callerSession: string | null): Promise<RestartPlan> {
+  const record = findDaemonRecord(daemonRecordFile);
+
+  const paths = new Set([daemonSocketPath, ...(record === null ? [] : [record.socketPath])]);
+
+  let probe: Probe | null = null;
+
+  for (const path of paths) {
+    probe = await tryProbe(path);
+
+    if (probe !== null) {
+      break;
+    }
+  }
+
+  const pid =
+    probe === null
+      ? pickLivePID(record?.pid ?? null, findPidFilePID(daemonPidFile))
+      : (pickStaleDaemonPID({
+          socketPath: probe.socketPath,
+          record,
+          pidFileSocketPath: daemonSocketPath,
+          pidFilePID: findPidFilePID(daemonPidFile),
+        }) ?? pickLivePID(record?.pid ?? null, null));
+
+  return {
+    pid,
+    socketPath: probe?.socketPath ?? null,
+    answer: probe?.answer ?? null,
+    sessions: probe?.sessions ?? null,
+    callerSession,
+    replacement: await planReplacement(pid),
+  };
+}
+
+function pickLivePID(...candidates: readonly (number | null)[]): number | null {
+  return (
+    candidates.find(
+      (pid) => pid !== null && isProcessAlive(pid) && isDaemonProcess(pid, stateDir),
+    ) ?? null
+  );
+}
+
+async function planReplacement(pid: number | null): Promise<ReplacementPlan> {
+  const unit = pid === null ? null : await findRestartUnit(pid);
+
+  if (unit === null) {
+    return { kind: 'plain', build: getBuild(), command: buildATCCommand(['daemon']).join(' ') };
+  }
+
+  const shown = await runSystemctl(['show', '-p', 'ExecStart', '--value', unit]);
+
+  const execPath = /path=(?<path>[^\s;]+)/.exec(shown.stdout)?.groups?.['path'];
+
+  return {
+    kind: 'unit',
+    unit,
+    execStart: execPath ?? (shown.code === 0 && shown.stdout !== '' ? shown.stdout : null),
+  };
+}
+
+async function tryProbe(socketPath: string): Promise<Probe | null> {
+  let client: DaemonClient;
+
+  try {
+    client = await DaemonClient.open(socketPath);
+  } catch {
+    return null;
+  }
+
+  // A daemon that accepts the socket and never answers must not hang the
+  // restart: closing the client rejects every request still waiting.
+  const expiry = setTimeout(() => {
+    client.stop();
+  }, PROBE_DEADLINE_MS);
+
+  try {
+    const hello = await client.sendHello(getBuild());
+
+    const build = hello['daemon'];
+
+    return {
+      socketPath,
+      answer: {
+        kind: 'ok',
+        build: typeof build === 'string' ? build : 'unknown',
+        protocol: PROTOCOL_V,
+      },
+      sessions: await tryListSessions(client),
+    };
+  } catch (error) {
+    if (error instanceof DaemonError && error.code === 'protocol_mismatch') {
+      return { socketPath, answer: { kind: 'refused', message: error.message }, sessions: null };
+    }
+
+    return null;
+  } finally {
+    clearTimeout(expiry);
+
+    client.stop();
+  }
+}
+
+async function tryListSessions(
+  client: Pick<DaemonClient, 'sendRequest'>,
+): Promise<PlanSession[] | null> {
+  try {
+    const listed = await client.sendRequest('session.list');
+
+    const sessions = listed['sessions'];
+
+    if (!Array.isArray(sessions)) {
+      return null;
+    }
+
+    return sessions
+      .filter((entry) => isRecord(entry))
+      .map((entry) => ({
+        id: String(entry['id']),
+        name: String(entry['name']),
+        state: String(entry['state']),
+      }));
+  } catch {
+    return null;
+  }
+}

package/src/daemon/build-scoped-context.ts

@@ -137,8 +137,10 @@ export function buildScopedContext(
 
     // A session out of reach answers before any confirm token is handed
     // out or taken, so its host is never touched.
-    forgetSession: (id, confirmToken) =>
-      canSee(id) ? ctx.forgetSession(id, confirmToken) : Promise.resolve('missing' as const),
+    forgetSession: (id, confirmToken, refuse) =>
+      canSee(id)
+        ? ctx.forgetSession(id, confirmToken, refuse)
+        : Promise.resolve('missing' as const),
     revokeSessionAuth: (id) => (canSee(id) ? ctx.revokeSessionAuth(id) : Promise.resolve(false)),
     updateSessionAuth: (id) => (canSee(id) ? ctx.updateSessionAuth(id) : Promise.resolve(null)),
     updateSession: (id, name, pinned) => canSee(id) && ctx.updateSession(id, name, pinned),

package/src/daemon/daemon-connection.ts

@@ -650,7 +650,10 @@ export class DaemonConnection {
 
         const id = parsed.data.session;
 
-        const forgotten = await ctx.forgetSession(id, parsed.data.confirmToken);
+        const forgotten = await ctx.forgetSession(id, parsed.data.confirmToken, {
+          pinned: parsed.data.refusePinned === true,
+          live: parsed.data.refuseLive === true,
+        });
 
         if (forgotten === 'missing') {
           this.sendErr(req.id, 'no_such_session', `no session '${id}'`);

package/src/daemon/daemon-context.ts

@@ -63,6 +63,15 @@ type ForgetResult =
   | { readonly confirmToken: string; readonly expiresAt: number }
   | { readonly forgotten: true; readonly destroyed: boolean };
 
+/**
+ * The states a forget refuses to act on: a pinned session, or a sub-session
+ * of a pinned one, and a live session.
+ */
+interface ForgetRefusals {
+  readonly pinned: boolean;
+  readonly live: boolean;
+}
+
 // A transcript page with the file it came from, so a cursor into a replaced
 // transcript is distinguishable from one into a grown transcript.
 interface SessionTranscriptRead {
@@ -223,10 +232,12 @@ export interface DaemonContext {
   readonly killSession: (id: SessionID) => Promise<boolean>;
 
   // Forgets a session, or answers with the token a forget that destroys a
-  // host must carry. Throws the refusal for a token it does not take.
+  // host must carry. Throws the refusal for a token it does not take, and
+  // for a pinned or live session when the caller asks it to refuse one.
   readonly forgetSession: (
     id: SessionID,
     confirmToken: string | undefined,
+    refuse: ForgetRefusals,
   ) => Promise<ForgetResult | 'missing'>;
 
   // Withdraws the grants of the runtime auth binding on a session's host,

package/src/daemon/daemon.ts

@@ -12,11 +12,13 @@ import type { EventMsg } from '../protocol/protocol';
 import type { SpawnWorkspaceSource } from '../protocol/request-param-schemas';
 import type { SessionState } from '../protocol/session-state';
 import type { AgentID } from '../shared/agent-id';
+import { claimDaemonLock } from '../shared/claim-daemon-lock';
 import type { HooksConfig } from '../shared/collect-hooks';
 import type { TargetConfigError } from '../shared/collect-targets';
 import type { InvalidGitTransports } from '../shared/collect-workspaces-config';
 import { DEFAULT_GIT_TRANSPORTS } from '../shared/default-git-transports';
 import { findDaemonRecord } from '../shared/find-daemon-record';
+import { loadListenerTokens } from '../shared/load-listener-tokens';
 import type { MessageID } from '../shared/message-id';
 import { isRecord } from '../shared/report';
 import { resolveHomeDir } from '../shared/resolve-home-dir';
@@ -50,7 +52,6 @@ import { buildSessionReportEvent } from './build-session-report-event';
 import { buildTargetAccess } from './build-target-access';
 import { buildTargetForbiddenError } from './build-target-forbidden-error';
 import { buildTargetList } from './build-target-list';
-import { claimDaemonLock } from './claim-daemon-lock';
 import { createNonBlockingLog } from './create-non-blocking-log';
 import type { NonBlockingLog } from './create-non-blocking-log';
 import { DaemonConnection } from './daemon-connection';
@@ -70,7 +71,6 @@ import { IdempotencyLedger } from './idempotency-ledger';
 import { isAllowedListenHost } from './is-allowed-listen-host';
 import { isOwnHookEvent } from './is-own-hook-event';
 import { isTreeInReach } from './is-tree-in-reach';
-import { loadListenerTokens } from './load-listener-tokens';
 import { loadTranscriptPage } from './load-transcript-page';
 import { makeHookRunner } from './make-hook-runner';
 import type { HookScope } from './make-hook-runner';
@@ -1112,7 +1112,11 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       runtime.pendingLastUsed = true;
     }
 
-    void store.recordSpawnDir(s.cwd, { target: s.target, targetIdentity: s.targetIdentity });
+    void store.recordSpawnDir(
+      s.cwd,
+      { target: s.target, targetIdentity: s.targetIdentity },
+      Date.now(),
+    );
 
     return getDescriptor(mgr, s.id);
   };
@@ -1573,13 +1577,31 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     // A forget on a target that cannot destroy its host forgets at once. On
     // one that can, a forget without a token checks the target and answers
     // with a token, and the forget that carries the token destroys the host.
-    forgetSession: async (id, confirmToken) => {
+    // A refusal the caller asks for reads the session in the same step that
+    // starts the forget, so a pin or revive that lands first is seen.
+    forgetSession: async (id, confirmToken, refuse) => {
       const s = mgr.sessions.find((x) => x.id === id);
 
       if (s === undefined) {
         return 'missing';
       }
 
+      if (refuse.pinned && (s.pinned || mgr.sessions.some((x) => x.id === s.parent && x.pinned))) {
+        throw new DaemonError(
+          'session_pinned',
+          `session ${id} is pinned, or is a sub-session of a pinned session; unpin it before forgetting it`,
+          { session: id },
+        );
+      }
+
+      if (refuse.live && (s.pty !== null || (s.kind === 'headless' && s.state !== 'exited'))) {
+        throw new DaemonError(
+          'session_live',
+          `session ${id} is live; stop it before forgetting it`,
+          { session: id },
+        );
+      }
+
       if (mgr.findProvider(s)?.capabilities.destroy === true) {
         mgr.requireExecution(s, 'destroy');