@zgeoff/atc 3.3.0 → 3.4.0

package/src/daemon/make-hook-runner.ts

@@ -12,20 +12,68 @@ export interface HookScope {
 
 export type RunHooks = (event: EventMsg, scope: HookScope | null) => void;
 
+/**
+ * How one hook run ended: its exit code, or the signal that ended it, or
+ * neither when the command failed to spawn.
+ */
+export interface HookOutcome {
+  readonly command: string;
+  readonly exitCode: number | null;
+  readonly signalCode: string | null;
+}
+
+interface HookRunnerOptions {
+  /**
+   * Runs once per call, after every hook the call started has exited or
+   * failed to spawn, with each one's outcome in configured order; a call
+   * that starts none settles at once with none.
+   */
+  readonly onSettled?: (event: EventMsg, outcomes: readonly HookOutcome[]) => void;
+
+  /**
+   * Arms the kill of a run that outlives its timeout and returns the
+   * function that disarms it; a timer that never keeps the process alive
+   * when unset.
+   */
+  readonly scheduleKill?: (kill: () => void, timeoutMs: number) => () => void;
+}
+
 /**
  * Builds the daemon's hook runner. Each call fires every configured command
  * for the event's name, with the wire-event JSON on stdin and the event name
  * in `ATC_EVENT`. Hooks are observational and fire-and-forget: the daemon
  * never waits on one, a run past its timeout is killed, and a nonzero exit
- * or spawn failure is logged to stderr and otherwise ignored.
+ * or spawn failure is logged to stderr and otherwise ignored. An event with
+ * no matching hook does no work beyond the match.
  */
-export function makeHookRunner(hooks: HooksConfig): RunHooks {
+export function makeHookRunner(hooks: HooksConfig, options: HookRunnerOptions = {}): RunHooks {
+  const onSettled = options.onSettled ?? (() => {});
+  const scheduleKill = options.scheduleKill ?? scheduleUnrefKill;
+
   return (event, scope) => {
-    for (const entry of hooks[event.ev] ?? []) {
-      if (entry.dir === undefined || isInScope(entry.dir, scope)) {
-        runHook(entry, event);
-      }
+    const matched = (hooks[event.ev] ?? []).filter(
+      (entry) => entry.dir === undefined || isInScope(entry.dir, scope),
+    );
+
+    if (matched.length === 0) {
+      onSettled(event, []);
+
+      return;
     }
+
+    void runHooks(matched, event, scheduleKill, onSettled);
+  };
+}
+
+// Unref'd so an in-flight hook never keeps the daemon process alive; a hook
+// orphaned by daemon exit is on its own.
+function scheduleUnrefKill(kill: () => void, timeoutMs: number): () => void {
+  const timer = setTimeout(kill, timeoutMs);
+
+  timer.unref();
+
+  return () => {
+    clearTimeout(timer);
   };
 }
 
@@ -44,9 +92,25 @@ function isUnderDir(candidate: string, dir: string): boolean {
   return candidate === dir || candidate.startsWith(prefix);
 }
 
+// Starts every hook at once and reports their outcomes once all have ended.
+async function runHooks(
+  entries: readonly HookEntry[],
+  event: EventMsg,
+  scheduleKill: (kill: () => void, timeoutMs: number) => () => void,
+  onSettled: (event: EventMsg, outcomes: readonly HookOutcome[]) => void,
+): Promise<void> {
+  const outcomes = await Promise.all(entries.map((entry) => runHook(entry, event, scheduleKill)));
+
+  onSettled(event, outcomes);
+}
+
 const DEFAULT_TIMEOUT_MS = 10_000;
 
-function runHook(entry: HookEntry, event: EventMsg): void {
+async function runHook(
+  entry: HookEntry,
+  event: EventMsg,
+  scheduleKill: (kill: () => void, timeoutMs: number) => () => void,
+): Promise<HookOutcome> {
   let proc: ReturnType<typeof Bun.spawn>;
 
   try {
@@ -59,26 +123,22 @@ function runHook(entry: HookEntry, event: EventMsg): void {
   } catch (error) {
     console.error(`atc hook for ${event.ev} failed to spawn: ${String(error)}`);
 
-    return;
+    return { command: entry.command, exitCode: null, signalCode: null };
   }
 
-  // Unref'd so an in-flight hook never keeps the daemon process alive; a
-  // hook orphaned by daemon exit is on its own.
-  const timer = setTimeout(() => {
+  const disarm = scheduleKill(() => {
     proc.kill();
   }, entry.timeout ?? DEFAULT_TIMEOUT_MS);
 
-  timer.unref();
-
-  void (async () => {
-    try {
-      const code = await proc.exited;
+  try {
+    const code = await proc.exited;
 
-      if (code !== 0) {
-        console.error(`atc hook for ${event.ev} exited ${code}`);
-      }
-    } finally {
-      clearTimeout(timer);
+    if (code !== 0) {
+      console.error(`atc hook for ${event.ev} exited ${code}`);
     }
-  })();
+
+    return { command: entry.command, exitCode: proc.exitCode, signalCode: proc.signalCode };
+  } finally {
+    disarm();
+  }
 }

package/src/daemon/permission-registry.ts

@@ -1,4 +1,6 @@
 import type { SessionID } from '../shared/session-id';
+import { systemClock } from '../shared/system-clock';
+import type { Clock } from '../shared/system-clock';
 
 export interface PermissionRequest {
   readonly id: string;
@@ -11,7 +13,7 @@ export type AnswerResult = 'ok' | 'already_answered' | 'unsupported' | 'unknown'
 
 interface PendingRequest {
   readonly req: PermissionRequest;
-  readonly timer: ReturnType<typeof setTimeout>;
+  readonly cancelTimeout: () => void;
 }
 
 /**
@@ -28,6 +30,8 @@ export class PermissionRegistry {
 
   private readonly timeoutMs: number;
 
+  private readonly clock: Clock;
+
   private readonly pending = new Map<string, PendingRequest>();
 
   // Each answered request's session, kept to tell already_answered from
@@ -36,19 +40,21 @@ export class PermissionRegistry {
 
   private counter = 0;
 
-  constructor(timeoutMs = 60_000) {
+  // The clock that times each request's timeout.
+  constructor(timeoutMs = 60_000, clock: Clock = systemClock) {
     this.timeoutMs = timeoutMs;
+    this.clock = clock;
   }
 
   open(sessionID: SessionID, message: string, respondable: boolean): PermissionRequest {
     const id = `p${++this.counter}`;
     const req: PermissionRequest = { id, sessionID, message, respondable };
 
-    const timer = setTimeout(() => {
+    const cancelTimeout = this.clock.schedule(() => {
       this.removeRequest(id, 'deny');
     }, this.timeoutMs);
 
-    this.pending.set(id, { req, timer });
+    this.pending.set(id, { req, cancelTimeout });
     this.onRequested(req);
 
     return req;
@@ -97,8 +103,7 @@ export class PermissionRegistry {
       return;
     }
 
-    clearTimeout(entry.timer);
-
+    entry.cancelTimeout();
     this.pending.delete(id);
     this.resolved.set(id, entry.req.sessionID);
 

package/src/daemon/restore-fleet.ts

@@ -1,5 +1,7 @@
 import { DaemonError } from '../protocol/daemon-error';
 import type { SessionID } from '../shared/session-id';
+import { systemClock } from '../shared/system-clock';
+import type { Clock } from '../shared/system-clock';
 import type { FleetEntry } from '../store/fleet-entry';
 import type { StateStore } from '../store/state-store';
 import type { SessionRuntime } from './session-runtime';
@@ -16,15 +18,34 @@ export interface RestoreFleetParams {
   // it has booted before moving on regardless; zero waits on the signal
   // alone.
   readonly capMs: number;
+
+  // Whether the daemon has begun to stop; never when unset. A restore
+  // checks it before each terminal it starts and after each one it has
+  // started, and starts no more once it holds.
+  readonly isStopped?: () => boolean;
+
+  // The clock behind the per-session cap; the wall clock when unset.
+  readonly clock?: Clock;
+}
+
+// How the staggered terminal adoption behind a restore ended: every queued
+// session was tried, the daemon began to stop first, or a revive threw.
+type RestoreOutcome = 'finished' | 'stopped' | 'failed';
+
+// How many sessions a restore registered, and how its staggered terminal
+// adoption ended.
+export interface RestoreSettled {
+  readonly restored: number;
+  readonly outcome: RestoreOutcome;
 }
 
 export interface RestoreFleetResult {
   // How many sessions the restore registered.
   readonly restored: number;
 
-  // Resolves once the staggered terminal adoption behind the restore has
-  // finished, also when it failed, and at once when nothing was queued.
-  readonly settled: Promise<void>;
+  // Resolves with how the staggered terminal adoption behind the restore
+  // ended, at once when nothing was queued.
+  readonly settled: Promise<RestoreOutcome>;
 }
 
 /**
@@ -41,6 +62,8 @@ export async function restoreFleet(params: RestoreFleetParams): Promise<RestoreF
   const cols = params.cols;
   const rows = params.rows;
   const capMs = params.capMs;
+  const clock = params.clock ?? systemClock;
+  const isStopped = () => params.isStopped?.() === true;
 
   const hasLiveSession = (entry: FleetEntry) =>
     mgr.sessions.some(
@@ -108,7 +131,7 @@ export async function restoreFleet(params: RestoreFleetParams): Promise<RestoreF
   // A session whose target refuses to start a terminal stays listed without
   // one, and the restore moves on to the next.
   const adoptQueued = async (s: Session): Promise<boolean> => {
-    const adopted = await tryAdoptTerminal(mgr, s.id, cols, rows);
+    const adopted = await tryAdoptTerminal(mgr, s.id, cols, rows, () => !isStopped());
 
     return adopted !== null;
   };
@@ -128,29 +151,24 @@ export async function restoreFleet(params: RestoreFleetParams): Promise<RestoreF
       return Promise.resolve();
     }
 
+    // The runtime's dispose releases the waiter, which cancels the cap.
     const settled = Promise.withResolvers<void>();
-    const timer = capMs > 0 ? setTimeout(settled.resolve, capMs) : undefined;
+    const cancelCap = capMs > 0 ? clock.schedule(settled.resolve, capMs) : null;
 
     runtime.bootWaiter = settled.resolve;
-    runtime.bootTimer = timer;
 
     return (async () => {
       await settled.promise;
 
       runtime.bootWaiter = null;
-
-      if (timer !== undefined) {
-        clearTimeout(timer);
-
-        runtime.bootTimer = undefined;
-      }
+      cancelCap?.();
     })();
   };
 
   const [first, ...rest] = queued;
 
   if (first === undefined) {
-    return { restored: registered.length, settled: Promise.resolve() };
+    return { restored: registered.length, settled: Promise.resolve('finished') };
   }
 
   // The first terminal attaches before the restore answers so a caller can
@@ -161,10 +179,14 @@ export async function restoreFleet(params: RestoreFleetParams): Promise<RestoreF
   // A session whose revive fails is logged and left without a terminal,
   // and the restore moves on: one host's failure never keeps the sessions on
   // other hosts down, and a later one's runs where nothing awaits it.
+  let failed = false;
+
   const tryAdoptQueued = async (s: Session): Promise<boolean> => {
     try {
       return await adoptQueued(s);
     } catch (error) {
+      failed = true;
+
       mgr.log(
         `atc could not revive session ${s.id} (${error instanceof Error ? error.message : String(error)})`,
       );
@@ -173,7 +195,9 @@ export async function restoreFleet(params: RestoreFleetParams): Promise<RestoreF
     }
   };
 
-  const adoptRest = async (previous: Session | null) => {
+  // A daemon that stopped before a terminal started, or while one started
+  // or the previous session booted, starts no more terminals.
+  const adoptRest = async (previous: Session | null): Promise<RestoreOutcome> => {
     let prev = previous;
 
     for (const s of rest) {
@@ -181,23 +205,43 @@ export async function restoreFleet(params: RestoreFleetParams): Promise<RestoreF
         await waitForBoot(prev.id);
       }
 
+      if (isStopped()) {
+        return 'stopped';
+      }
+
       const booted = await tryAdoptQueued(s);
 
+      if (isStopped()) {
+        return 'stopped';
+      }
+
       prev = booted ? s : null;
     }
+
+    return failed ? 'failed' : 'finished';
   };
 
+  if (isStopped()) {
+    return { restored: registered.length, settled: Promise.resolve('stopped') };
+  }
+
   const firstAdopted = await tryAdoptQueued(first);
 
+  if (isStopped()) {
+    return { restored: registered.length, settled: Promise.resolve('stopped') };
+  }
+
   const firstBooted = firstAdopted ? first : null;
 
-  const adoptRestLogged = async (): Promise<void> => {
+  const adoptRestLogged = async (): Promise<RestoreOutcome> => {
     try {
-      await adoptRest(firstBooted);
+      return await adoptRest(firstBooted);
     } catch (error) {
       mgr.log(
         `atc could not finish restoring the fleet (${error instanceof Error ? error.message : String(error)})`,
       );
+
+      return 'failed';
     }
   };
 
@@ -247,9 +291,10 @@ async function tryAdoptTerminal(
   id: SessionID,
   cols: number,
   rows: number,
+  canProceed: () => boolean,
 ): Promise<Session | null> {
   try {
-    return await mgr.adoptTerminal(id, cols, rows);
+    return await mgr.adoptTerminal(id, cols, rows, canProceed);
   } catch (error) {
     if (error instanceof DaemonError && REFUSED_ADOPT_CODES.has(error.code)) {
       return null;

package/src/daemon/run-eject-handoff.ts

@@ -1,12 +1,21 @@
 import type { SessionID } from '../shared/session-id';
 import type { SessionRuntime } from './session-runtime';
 
+/**
+ * Starts a timer that runs `onTimeout` after `ms` milliseconds, returning
+ * what cancels it.
+ */
+export type SettleScheduler = (onTimeout: () => void, ms: number) => () => void;
+
 export interface EjectHandoffParams {
   readonly sessionID: SessionID;
   readonly prompt: string;
   readonly settleMs: number;
   readonly runtime: SessionRuntime;
   readonly startHeadlessTurn: (sessionID: SessionID, prompt: string) => boolean;
+
+  // Starts the settle timer; defaults to a real `setTimeout`.
+  readonly scheduleSettle?: SettleScheduler | undefined;
 }
 
 /**
@@ -17,17 +26,26 @@ export interface EjectHandoffParams {
  */
 export function runEjectHandoff(params: EjectHandoffParams): void {
   const settled = Promise.withResolvers<void>();
-  const timer = setTimeout(settled.resolve, params.settleMs);
+  const scheduleSettle = params.scheduleSettle ?? scheduleRealSettle;
+  const cancelSettle = scheduleSettle(settled.resolve, params.settleMs);
 
   params.runtime.pendingEject = settled.resolve;
 
   void (async () => {
     await settled.promise;
 
-    clearTimeout(timer);
+    cancelSettle();
 
     params.runtime.pendingEject = null;
 
     params.startHeadlessTurn(params.sessionID, params.prompt);
   })();
 }
+
+function scheduleRealSettle(onTimeout: () => void, ms: number): () => void {
+  const timer = setTimeout(onTimeout, ms);
+
+  return () => {
+    clearTimeout(timer);
+  };
+}

package/src/daemon/sessions.ts

@@ -3010,6 +3010,12 @@ export class SessionManager {
     }
   }
 
+  // A harness detaches once, so a disposal after a detach of every harness
+  // lets go of nothing again.
+  [Symbol.dispose]() {
+    this.detachAll();
+  }
+
   private emitChange() {
     this.writeStatus();
     this.onChange();

package/src/daemon/start-tcp-listener.ts

@@ -1,3 +1,4 @@
+import type { Clock } from '../shared/system-clock';
 import type { DaemonConnection, TCPPeer } from './daemon-connection';
 import { findTokenFingerprint } from './find-token-fingerprint';
 import { formatLogField } from './format-log-field';
@@ -30,12 +31,13 @@ interface TCPListenerOptions {
   readonly closeConnection: (connection: DaemonConnection) => void;
 
   // Where the listener start and each refusal are logged, one line at a
-  // time, with the clock and the window that summarize repeated refusals
-  // from one peer.
+  // time, and the window that summarizes repeated refusals from one peer.
   readonly log: (line: string) => void;
-  readonly now: () => number;
   readonly refusalLogIntervalMs: number;
 
+  // The clock that times failures, handshake delays, and refusal windows.
+  readonly clock: Clock;
+
   // How many peers and kinds of refusal the refusal log tracks at once.
   readonly maxRefusalWindows: number;
 }
@@ -69,7 +71,7 @@ export function startTCPListener(opts: TCPListenerOptions): TCPListener {
 
   const refusals = new RefusalLog({
     log: opts.log,
-    now: opts.now,
+    now: opts.clock.now,
     intervalMs: opts.refusalLogIntervalMs,
     maxWindows: opts.maxRefusalWindows,
   });
@@ -91,10 +93,10 @@ export function startTCPListener(opts: TCPListenerOptions): TCPListener {
 
         const peer: TCPPeer = {
           verifyHandshake: async (presented) => {
-            const delay = throttle.getDelay(address, Date.now());
+            const delay = throttle.getDelay(address, opts.clock.now());
 
             if (delay > 0 && delayed >= opts.maxDelayedHandshakes) {
-              throttle.recordFailure(address, Date.now());
+              throttle.recordFailure(address, opts.clock.now());
 
               refusals.record({
                 event: 'handshake_refused',
@@ -110,13 +112,13 @@ export function startTCPListener(opts: TCPListenerOptions): TCPListener {
 
               const elapsed = Promise.withResolvers<'elapsed'>();
 
-              const timer = setTimeout(() => {
+              const cancel = opts.clock.schedule(() => {
                 elapsed.resolve('elapsed');
               }, delay);
 
               const waited = await Promise.race([elapsed.promise, closed.promise]);
 
-              clearTimeout(timer);
+              cancel();
 
               delayed--;
 
@@ -137,7 +139,7 @@ export function startTCPListener(opts: TCPListenerOptions): TCPListener {
                 : findTokenFingerprint(tokens, presented);
 
             if (fingerprint === null) {
-              throttle.recordFailure(address, Date.now());
+              throttle.recordFailure(address, opts.clock.now());
 
               refusals.record({
                 event: 'handshake_refused',
@@ -149,7 +151,7 @@ export function startTCPListener(opts: TCPListenerOptions): TCPListener {
             return fingerprint;
           },
           recordFailure: (reason) => {
-            throttle.recordFailure(address, Date.now());
+            throttle.recordFailure(address, opts.clock.now());
             refusals.record({ event: 'handshake_refused', peer: address, reason });
           },
           recordRefusedPrincipal: () => {

package/src/federation/daemon-caller.ts

@@ -32,6 +32,12 @@ export interface DaemonHello {
   readonly retentionMs: number | null;
 }
 
+/**
+ * Runs `onTimeout` once `ms` milliseconds have passed, and returns what
+ * cancels it.
+ */
+export type TimeoutScheduler = (onTimeout: () => void, ms: number) => () => void;
+
 interface DaemonCallerOptions {
   readonly daemon: RegistryDaemon;
   readonly build: string;
@@ -43,6 +49,9 @@ interface DaemonCallerOptions {
   // a sent request may wait for its response.
   readonly connectTimeoutMs?: number;
   readonly responseTimeoutMs?: number;
+
+  // Starts the connect and response timers; real timers when unset.
+  readonly scheduleTimeout?: TimeoutScheduler;
 }
 
 // One handshaken connection and what its handshake returned.
@@ -253,7 +262,7 @@ export class DaemonCaller {
     // gets its own wait on top of the response time.
     const waitMs = typeof sent['waitMs'] === 'number' && sent['waitMs'] > 0 ? sent['waitMs'] : 0;
 
-    const timer = setTimeout(
+    const cancelTimeout = this.scheduleTimeout(
       () => {
         timeout.resolve('timeout');
       },
@@ -272,7 +281,7 @@ export class DaemonCaller {
 
     const raced = await Promise.race([settled, timeout.promise]);
 
-    clearTimeout(timer);
+    cancelTimeout();
 
     if (raced === 'timeout') {
       this.closed.add(channel);
@@ -306,6 +315,10 @@ export class DaemonCaller {
     }
   }
 
+  private scheduleTimeout(onTimeout: () => void, ms: number): () => void {
+    return (this.opts.scheduleTimeout ?? scheduleRealTimeout)(onTimeout, ms);
+  }
+
   private buildOutcomeUnknown(m: string): DaemonError {
     return new DaemonError(
       'outcome_unknown',
@@ -340,7 +353,7 @@ export class DaemonCaller {
 
     const timeout = Promise.withResolvers<'timeout'>();
 
-    const timer = setTimeout(() => {
+    const cancelTimeout = this.scheduleTimeout(() => {
       timeout.resolve('timeout');
     }, this.opts.connectTimeoutMs ?? CONNECT_TIMEOUT_MS);
 
@@ -396,7 +409,7 @@ export class DaemonCaller {
       resetConnection();
       throw error;
     } finally {
-      clearTimeout(timer);
+      cancelTimeout();
     }
   }
 
@@ -471,3 +484,11 @@ function buildDaemonHello(
     retentionMs: typeof retention === 'number' && retention > 0 ? retention : null,
   };
 }
+
+function scheduleRealTimeout(onTimeout: () => void, ms: number): () => void {
+  const timer = setTimeout(onTimeout, ms);
+
+  return () => {
+    clearTimeout(timer);
+  };
+}

package/src/federation/daemon-pool.ts

@@ -1,5 +1,5 @@
 import { DaemonCaller } from './daemon-caller';
-import type { GatewayChannel } from './daemon-caller';
+import type { GatewayChannel, TimeoutScheduler } from './daemon-caller';
 import type { GatewayRegistry, RegistryDaemon } from './types';
 
 interface DaemonPoolOptions {
@@ -8,6 +8,7 @@ interface DaemonPoolOptions {
   readonly openChannel: (address: RegistryDaemon['address']) => Promise<GatewayChannel>;
   readonly connectTimeoutMs?: number;
   readonly responseTimeoutMs?: number;
+  readonly scheduleTimeout?: TimeoutScheduler;
 }
 
 /**
@@ -31,6 +32,7 @@ export class DaemonPool {
           ...(opts.responseTimeoutMs === undefined
             ? {}
             : { responseTimeoutMs: opts.responseTimeoutMs }),
+          ...(opts.scheduleTimeout === undefined ? {} : { scheduleTimeout: opts.scheduleTimeout }),
         }),
       ]),
     );

package/src/federation/open-gateway-caller.ts

@@ -1,4 +1,4 @@
-import type { GatewayChannel } from './daemon-caller';
+import type { GatewayChannel, TimeoutScheduler } from './daemon-caller';
 import { DaemonPool } from './daemon-pool';
 import { GatewayStore } from './gateway-store';
 import { RoutingCaller } from './routing-caller';
@@ -16,6 +16,10 @@ interface GatewayCallerOptions {
 
   // How long each daemon may take to answer a call asked of every daemon.
   readonly fanOutTimeoutMs?: number;
+
+  // Starts each daemon connection's connect and response timers; real
+  // timers when unset.
+  readonly scheduleTimeout?: TimeoutScheduler;
 }
 
 /**
@@ -35,6 +39,7 @@ export function openGatewayCaller(opts: GatewayCallerOptions): {
     registry: opts.registry,
     build: opts.build,
     openChannel: opts.openChannel,
+    ...(opts.scheduleTimeout === undefined ? {} : { scheduleTimeout: opts.scheduleTimeout }),
   });
 
   const caller = new RoutingCaller({