@zgeoff/atc 3.3.1 → 3.4.0

package/src/clients.ts

@@ -18,13 +18,42 @@ interface ClientsTarget {
   readonly command: string;
 }
 
+// Where the command's lines and its exit code go, and how it exits: the
+// console and the process by default.
+interface ClientsIO {
+  readonly print: (line: string) => void;
+  readonly printError: (line: string) => void;
+  readonly setExitCode: (code: number) => void;
+  readonly exit: (code: number) => void;
+}
+
+const PROCESS_IO: ClientsIO = {
+  print: (line) => {
+    console.log(line);
+  },
+  printError: (line) => {
+    console.error(line);
+  },
+  setExitCode: (code) => {
+    process.exitCode = code;
+  },
+  exit: (code) => {
+    process.exit(code);
+  },
+};
+
 /**
  * Runs `atc clients` or `atc-gateway clients`: lists the clients that may
  * connect to the MCP HTTP server, adds one, or removes one along with every
  * token and consent it holds. It opens the authorization server's database
- * directly, so it works whether or not the server is running.
+ * directly, so it works whether or not the server is running. A refused
+ * add exits 1 at once, and a refused remove sets exit code 1.
  */
-export async function runClients(action: ClientsAction, target: ClientsTarget): Promise<void> {
+export async function runClients(
+  action: ClientsAction,
+  target: ClientsTarget,
+  io: ClientsIO = PROCESS_IO,
+): Promise<void> {
   if (action.kind === 'add') {
     const refused = action.redirectURIs.find((uri) => !isAllowedRedirectURI(uri));
 
@@ -34,8 +63,10 @@ export async function runClients(action: ClientsAction, target: ClientsTarget):
           ? `${target.command} add: give at least one --redirect-uri`
           : `${target.command} add: '${refused}' is not a redirect URI atc accepts; use https, or http on a loopback host, with no fragment`;
 
-      console.error(message);
-      process.exit(1);
+      io.printError(message);
+      io.exit(1);
+
+      return;
     }
   }
 
@@ -51,7 +82,7 @@ export async function runClients(action: ClientsAction, target: ClientsTarget):
         body: { name, redirectURIs: [...action.redirectURIs] },
       });
 
-      console.log(`Added ${name}. Its client ID is ${created.clientID}`);
+      io.print(`Added ${name}. Its client ID is ${created.clientID}`);
 
       return;
     }
@@ -60,14 +91,13 @@ export async function runClients(action: ClientsAction, target: ClientsTarget):
       const removed = await removeClient(store.db, action.clientID);
 
       if (!removed) {
-        console.error(`${target.command} remove: no client has the ID '${action.clientID}'`);
-
-        process.exitCode = 1;
+        io.printError(`${target.command} remove: no client has the ID '${action.clientID}'`);
+        io.setExitCode(1);
 
         return;
       }
 
-      console.log(`Removed client ${action.clientID} and revoked every grant it held`);
+      io.print(`Removed client ${action.clientID} and revoked every grant it held`);
 
       return;
     }
@@ -75,13 +105,13 @@ export async function runClients(action: ClientsAction, target: ClientsTarget):
     const clients = await collectClients(store.db);
 
     if (clients.length === 0) {
-      console.log(`No clients. Add one with: ${target.command} add <name> --redirect-uri <uri>`);
+      io.print(`No clients. Add one with: ${target.command} add <name> --redirect-uri <uri>`);
 
       return;
     }
 
     for (const client of clients) {
-      console.log(`${client.clientID}  ${client.name}  ${client.redirectURIs.join(' ')}`);
+      io.print(`${client.clientID}  ${client.name}  ${client.redirectURIs.join(' ')}`);
     }
   } finally {
     await store.close();

package/src/daemon/daemon.ts

@@ -23,6 +23,8 @@ import type { MessageID } from '../shared/message-id';
 import { isRecord } from '../shared/report';
 import { resolveHomeDir } from '../shared/resolve-home-dir';
 import type { SessionID } from '../shared/session-id';
+import { systemClock } from '../shared/system-clock';
+import type { Clock } from '../shared/system-clock';
 import { toMessageID } from '../shared/to-message-id';
 import { truncateToBytes } from '../shared/truncate-to-bytes';
 import type { SourceProvider } from '../sources/types';
@@ -66,6 +68,7 @@ import { drainInbox } from './drain-inbox';
 import type { InboxSource } from './drain-inbox';
 import { EffectRemainsError } from './effect-remains-error';
 import { EventSignal } from './event-signal';
+import { formatLogField } from './format-log-field';
 import { startHookServer } from './hooks';
 import { IdempotencyLedger } from './idempotency-ledger';
 import { isAllowedListenHost } from './is-allowed-listen-host';
@@ -81,7 +84,9 @@ import { parseReport } from './parse-report';
 import { PermissionRegistry } from './permission-registry';
 import { requireGitTransports } from './require-git-transports';
 import { restoreFleet } from './restore-fleet';
+import type { RestoreSettled } from './restore-fleet';
 import { runEjectHandoff } from './run-eject-handoff';
+import type { SettleScheduler } from './run-eject-handoff';
 import { RuntimeAuthBinder } from './runtime-auth-binder';
 import { ScreenModel } from './screen-model';
 import { SessionRuntime } from './session-runtime';
@@ -154,6 +159,9 @@ export interface DaemonOptions {
   // before starting the headless run anyway.
   readonly ejectSettleMs?: number;
 
+  // Starts the eject settle timer; defaults to a real `setTimeout`.
+  readonly scheduleEjectSettle?: SettleScheduler;
+
   // A fleet-wide restore revives one session at a time, waiting for each to
   // report it has booted before starting the next so the machine is not
   // buried under a dozen simultaneous agent boots. This caps how long a
@@ -162,13 +170,43 @@ export interface DaemonOptions {
   readonly restoreBootTimeoutMs?: number;
   readonly tapGraceMs?: number;
 
+  // The clock the event trail stamps each hook event with, and the clock
+  // behind the tap grace window, the held-read waits, the time a note is
+  // reported at, the permission request timeouts, the restore's per-session
+  // boot cap, and the TCP listener's failure times, delays, and refusal
+  // windows; the wall clock when unset.
+  readonly clock?: Clock;
+
   // Whether the daemon restores the stored fleet by itself once it is
   // listening, when that fleet holds sessions; off when unset.
   readonly restoreFleetOnRestart?: boolean;
 
+  // Called once each fleet restore has ended its staggered terminal
+  // adoption, with how many sessions it registered and how it ended; a
+  // restore that threw before registering any reports none and failed.
+  readonly onRestoreSettled?: (settled: RestoreSettled) => void;
+
+  // Called when a start declines to restore the stored fleet by itself:
+  // the option is off, or the stored fleet holds no sessions.
+  readonly onRestoreSkipped?: (reason: 'disabled' | 'empty') => void;
+
+  // Called when a Report line changes nothing: a note from a session the
+  // fleet does not list, or an answer that matches no unanswered message
+  // the reporting session owns.
+  readonly onReportIgnored?: (sessionID: string, kind: 'note' | 'answered') => void;
+
   // How long a confirm token from `session.forget` stays usable.
   readonly forgetConfirmMs?: number;
 
+  // The clock confirm tokens from `session.forget` are minted and checked
+  // against, in epoch milliseconds; the wall clock when unset.
+  readonly forgetClock?: () => number;
+
+  // Called with a session's id when it reports its start and the daemon
+  // leaves the last-used agent as it is, because no spawn is waiting on
+  // that start.
+  readonly onLastUsedUnchanged?: (sessionID: SessionID) => void;
+
   // When set, a TCP listener serves the client protocol on this address to
   // peers whose handshake presents a token from the token file.
   readonly listen?: ListenOptions;
@@ -181,6 +219,11 @@ export interface DaemonOptions {
   // when unset.
   readonly log?: (line: string) => void;
 
+  // Called each time the daemon leaves a session's new output unjudged
+  // because no screen detector covers it: no agent in the fleet has one,
+  // the session's own agent has none, or the session is no longer listed.
+  readonly onDetectSkipped?: (sessionID: SessionID) => void;
+
   // The sources the spawn picker offers, in order, each built with the
   // services it uses; none when unset.
   readonly sources?: readonly SourceProvider[];
@@ -197,6 +240,11 @@ export interface DaemonOptions {
     root: string | null;
     targetRoots: ReadonlyMap<string, string>;
   }>;
+
+  // The home a git workspace without a directory lands under on a target
+  // on the daemon's own machine when no root applies; the daemon user's
+  // home when unset.
+  readonly homeDir?: string;
 }
 
 // The TCP listener's address and the file holding the tokens a handshake
@@ -214,10 +262,8 @@ interface ListenOptions {
   // when unset.
   readonly maxDelayedHandshakes?: number;
 
-  // The clock the refusal log reads, Date.now when unset, and how long it
-  // folds repeated refusals from one peer into one line; a minute when
-  // unset.
-  readonly now?: () => number;
+  // How long the refusal log folds repeated refusals from one peer into one
+  // line; a minute when unset.
   readonly refusalLogIntervalMs?: number;
 
   // How many peers and kinds of refusal the refusal log tracks at once;
@@ -226,11 +272,18 @@ interface ListenOptions {
 }
 
 export interface DaemonHandle {
+  // A second stop, or a disposal after one, waits on the first and stops
+  // nothing again.
   readonly stop: () => Promise<void>;
 
+  readonly [Symbol.asyncDispose]: () => Promise<void>;
+
   // How many client-protocol connections are open right now.
   readonly countClients: () => number;
 
+  // How many event reads are waiting right now for the trail to grow.
+  readonly countEventWaiters: () => number;
+
   // The port the TCP listener bound, or null without one.
   readonly listenPort: number | null;
 
@@ -292,6 +345,10 @@ const LOCK_WAIT_MS = 2000;
  */
 export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   let stopDaemon: (() => Promise<void>) | null = null;
+  const clock = opts.clock ?? systemClock;
+
+  // Set once the daemon begins to release what it holds.
+  let released = false;
 
   // The listener's address and tokens are checked before the daemon takes
   // anything, so a refused listener leaves no state behind.
@@ -406,8 +463,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     return s === undefined ? null : { cwd: s.cwd, repoRoot: s.repoRoot };
   };
 
-  const registry = new PermissionRegistry();
-  const eventSignal = new EventSignal();
+  const registry = new PermissionRegistry(undefined, clock);
+  const eventSignal = new EventSignal(clock);
 
   // A trail write that fails never fails the message request or report behind it.
   // Returns false when the entry was not written: a failed write, or a
@@ -563,9 +620,11 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     if (report.kind === 'note') {
       const sender = mgr.sessions.find((x) => x.id === e.atcId);
 
-      if (sender !== undefined) {
+      if (sender === undefined) {
+        opts.onReportIgnored?.(e.atcId, 'note');
+      } else {
         const capped = { ...report, text: truncateToBytes(report.text, ANSWER_BYTE_CAP) };
-        const reportedAt = Date.now();
+        const reportedAt = clock.now();
 
         const entry = buildReportTrailEntry(
           sender.id,
@@ -602,6 +661,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         report.turn,
       );
 
+      if (answered.length === 0) {
+        opts.onReportIgnored?.(e.atcId, 'answered');
+      }
+
       for (const record of answered) {
         await recordMessageStatus(e.atcId, record);
       }
@@ -612,6 +675,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   // quiesced, judge the serialized screen and flip running/needs_you.
   const scheduleDetect = (sessionID: SessionID) => {
     if (!mgr.hasScreenDetector) {
+      opts.onDetectSkipped?.(sessionID);
+
       return;
     }
 
@@ -619,6 +684,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     const detector = s === undefined ? null : (mgr.findAdapter(s.agent)?.screenDetector ?? null);
     const runtime = runtimes.get(sessionID);
 
+    if (detector === null) {
+      opts.onDetectSkipped?.(sessionID);
+    }
+
     if (detector === null || runtime === undefined) {
       return;
     }
@@ -852,7 +921,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   };
 
   const recordHookEvent = async (e: HookEvent, ev: Readonly<AdapterEvent> | null) => {
-    await store.recordEvent(e, ev);
+    await store.recordEvent(e, ev, clock.now());
 
     eventSignal.emit();
   };
@@ -883,7 +952,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     const runtime = runtimes.get(e.atcId);
 
     // A harness nested inside a session inherits its environment and reports
-    // under its id; only the harness atc started may change the session.
+    // under its id; only the harness atc started may change the session, and
+    // a line from any other is logged and dropped.
     if (
       before !== undefined &&
       runtime !== undefined &&
@@ -893,6 +963,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         runtime.hasAgentHookLines,
       )
     ) {
+      mgr.log(
+        `atc hook event=dropped session=${formatLogField(e.atcId)} agent=${formatLogField(e.agent ?? '')} hook=${formatLogField(e.event)}`,
+      );
+
       return;
     }
 
@@ -927,7 +1001,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     // waits on before booting the next one.
     if (kind === 'started') {
       if (runtime !== undefined) {
-        runtime.startedAt = Date.now();
+        runtime.startedAt = clock.now();
       }
 
       runtime?.bootWaiter?.();
@@ -936,6 +1010,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       if (started !== undefined && runtime !== undefined && runtime.pendingLastUsed) {
         runtime.pendingLastUsed = false;
         void store.writeLastUsedAgent(started.agent);
+      } else if (started !== undefined) {
+        opts.onLastUsedUnchanged?.(started.id);
       }
     }
   };
@@ -1210,9 +1286,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   const confirmTokens = new Map<string, ConfirmToken>();
 
   const forgetConfirmMs = opts.forgetConfirmMs ?? FORGET_CONFIRM_MS;
+  const forgetClock = opts.forgetClock ?? Date.now;
 
   const claimConfirmToken = (sessionID: SessionID, token: string) => {
-    const now = Date.now();
+    const now = forgetClock();
 
     for (const [held, entry] of confirmTokens) {
       if (entry.expiresAt + forgetConfirmMs < now) {
@@ -1268,7 +1345,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       runtime !== undefined &&
       runtime.startedAt !== null &&
       !runtime.tapAttached &&
-      Date.now() - runtime.startedAt >= (opts.tapGraceMs ?? TAP_GRACE_MS)
+      clock.now() - runtime.startedAt >= (opts.tapGraceMs ?? TAP_GRACE_MS)
     ) {
       return 'no_tap';
     }
@@ -1386,13 +1463,18 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
           cols,
           rows,
           capMs: opts.restoreBootTimeoutMs ?? 0,
+          isStopped: () => released,
+          clock,
         });
 
         restored.resolve(result.restored);
 
-        await result.settled;
+        const outcome = await result.settled;
+
+        opts.onRestoreSettled?.({ restored: result.restored, outcome });
       } catch (error) {
         restored.reject(error);
+        opts.onRestoreSettled?.({ restored: 0, outcome: 'failed' });
       } finally {
         fleetRestore = null;
       }
@@ -1480,7 +1562,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       buildDefaultWorkspaceDir(source, {
         root: opts.workspaceRoots?.targetRoots.get(target) ?? opts.workspaceRoots?.root ?? null,
         remote: mgr.requireExecution({ target, targetIdentity: null }, 'run').provider.remote,
-        home: resolveHomeDir(),
+        home: opts.homeDir ?? resolveHomeDir(),
       }),
     requireAgentTarget: (agent, target) => {
       mgr.requireAgentTarget(agent, target);
@@ -1607,7 +1689,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
         if (confirmToken === undefined) {
           const token = randomUUID();
-          const expiresAt = Date.now() + forgetConfirmMs;
+          const expiresAt = forgetClock() + forgetConfirmMs;
 
           confirmTokens.set(token, { session: id, expiresAt, used: false });
 
@@ -1663,6 +1745,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         sessionID: id,
         prompt,
         settleMs: opts.ejectSettleMs ?? 4000,
+        scheduleSettle: opts.scheduleEjectSettle,
         runtime,
         startHeadlessTurn: (sid, p) =>
           startHeadlessTurn(mgr, findRuntime, sid, p, recordHeadlessTurnEvent),
@@ -1911,7 +1994,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       return { path, page };
     },
     readEvents: async (afterID, limit, waitMs, sessionID, access) => {
-      const deadline = Date.now() + waitMs;
+      const deadline = clock.now() + waitMs;
 
       // A wake for an event the read leaves out (a heartbeat, or another
       // session's event under a session filter) loops back to wait out the
@@ -1936,7 +2019,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
           continue;
         }
 
-        const remaining = deadline - Date.now();
+        const remaining = deadline - clock.now();
 
         if (rows.length > 0 || remaining <= 0 || eventSignal.disposed) {
           const naming = collectNamingDescriptors(access);
@@ -2077,7 +2160,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       }
     },
     readMessage: async (messageID, waitMs) => {
-      const deadline = Date.now() + waitMs;
+      const deadline = clock.now() + waitMs;
       let initialStatus: MessageStatus | null = null;
 
       // Every message status change writes a trail entry and wakes the
@@ -2095,7 +2178,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
         initialStatus ??= view.record.status;
 
-        const remaining = deadline - Date.now();
+        const remaining = deadline - clock.now();
 
         if (
           view.record.status !== initialStatus ||
@@ -2139,6 +2222,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   // Releases everything the daemon holds except its listeners, on a stop
   // and on a start that fails once it holds the lock.
   const releaseResources = async () => {
+    released = true;
+
     clearInterval(idempotencySweep);
     eventsServer?.stop();
     reporter.stop(true);
@@ -2206,9 +2291,9 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         },
         closeConnection: detachConnection,
         log: listenerLog.log,
-        now: opts.listen.now ?? Date.now,
         refusalLogIntervalMs: opts.listen.refusalLogIntervalMs ?? REFUSAL_LOG_INTERVAL_MS,
         maxRefusalWindows: opts.listen.maxRefusalWindows ?? MAX_REFUSAL_WINDOWS,
+        clock,
       });
     } catch (error) {
       await releaseResources();
@@ -2262,22 +2347,41 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     );
   };
 
+  // The stop under way or done, which a quit, a second stop, and a disposal
+  // all wait on instead of releasing anything twice; a stop that fails is
+  // forgotten, so the next one retries the release.
+  let stopping: Promise<void> | null = null;
+
   stopDaemon = async () => {
-    // Ends each client itself so every peer sees the close: a stopped
-    // listener does not reliably end the connections it already accepted.
-    for (const client of clients) {
-      client.dispose();
-    }
+    stopping ??= (async () => {
+      // Ends each client itself so every peer sees the close: a stopped
+      // listener does not reliably end the connections it already accepted.
+      for (const client of clients) {
+        client.dispose();
+      }
+
+      tcpListener?.stop();
+      server.stop(true);
 
-    tcpListener?.stop();
-    server.stop(true);
+      await releaseResources();
+
+      // The stopped listener logs no more lines, so this writes the ones the
+      // log still holds, waiting no longer than the timeout when nothing
+      // reads stderr.
+      await listenerLog?.drain(LOG_DRAIN_TIMEOUT_MS);
+    })();
 
-    await releaseResources();
+    const current = stopping;
 
-    // The stopped listener logs no more lines, so this writes the ones the
-    // log still holds, waiting no longer than the timeout when nothing reads
-    // stderr.
-    await listenerLog?.drain(LOG_DRAIN_TIMEOUT_MS);
+    try {
+      await current;
+    } catch (error) {
+      if (stopping === current) {
+        stopping = null;
+      }
+
+      throw error;
+    }
   };
 
   writeDaemonRecord(recordPath, {
@@ -2306,12 +2410,18 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
     if (stored.length > 0) {
       void tryRunFleetRestore();
+    } else {
+      opts.onRestoreSkipped?.('empty');
     }
+  } else {
+    opts.onRestoreSkipped?.('disabled');
   }
 
   return {
     stop: stopDaemon,
+    [Symbol.asyncDispose]: stopDaemon,
     countClients: () => clients.size,
+    countEventWaiters: () => eventSignal.countWaiters(),
     listenPort: tcpListener?.port ?? null,
     refreshTokens,
   };

package/src/daemon/event-signal.ts

@@ -1,3 +1,6 @@
+import { systemClock } from '../shared/system-clock';
+import type { Clock } from '../shared/system-clock';
+
 /**
  * Wakes long-polling readers when the event trail grows. A reader notes the
  * signal's count before it queries and waits from that count, so an event
@@ -10,6 +13,13 @@ export class EventSignal {
 
   private readonly waiters = new Set<() => void>();
 
+  private readonly clock: Clock;
+
+  // The clock that times each wait's timeout.
+  constructor(clock: Clock = systemClock) {
+    this.clock = clock;
+  }
+
   emit(): void {
     this.generation += 1;
 
@@ -23,20 +33,26 @@ export class EventSignal {
 
     const deferred = Promise.withResolvers<void>();
 
-    const onWake = () => {
-      clearTimeout(timer);
+    // Holds the timer's cancel once it is scheduled.
+    let cancel: (() => void) | null = null;
 
+    const onWake = () => {
+      cancel?.();
       this.waiters.delete(onWake);
       deferred.resolve();
     };
 
-    const timer = setTimeout(onWake, timeoutMs);
+    cancel = this.clock.schedule(onWake, timeoutMs);
 
     this.waiters.add(onWake);
 
     return deferred.promise;
   }
 
+  countWaiters(): number {
+    return this.waiters.size;
+  }
+
   dispose(): void {
     this.disposed = true;
     this.generation += 1;