@zgeoff/atc 2.12.0 → 2.14.0

package/src/daemon/daemon.ts

@@ -1,10 +1,13 @@
+import { randomUUID } from 'node:crypto';
 import { unlinkSync, writeFileSync } from 'node:fs';
-import { hostname } from 'node:os';
+import { hostname, tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import type { AdapterEvent, AgentAdapter } from '../agents/agent-adapter';
+import { planTypedLineInput } from '../agents/plan-typed-line-input';
 import { DaemonError } from '../protocol/daemon-error';
 import { MAX_CHUNK, PROTOCOL_V } from '../protocol/protocol';
 import type { EventMsg } from '../protocol/protocol';
+import type { SpawnWorkspaceSource } from '../protocol/request-param-schemas';
 import type { HooksConfig } from '../shared/collect-hooks';
 import type { TargetConfigError } from '../shared/collect-targets';
 import { findDaemonRecord } from '../shared/find-daemon-record';
@@ -19,6 +22,7 @@ import type { MessageRecord, MessageStatus } from '../store/message-record';
 import { StateStore } from '../store/state-store';
 import type { EventScope } from '../store/state-store';
 import type { TrailEntry } from '../store/trail-entry';
+import type { SessionWorkspace } from '../store/workspace-materialization';
 import { ANSWER_BYTE_CAP } from './answer-byte-cap';
 import { AttachRegistry } from './attach-registry';
 import { buildAgentList } from './build-agent-list';
@@ -28,6 +32,7 @@ import type { ExecutionTarget } from './build-execution-targets';
 import { buildFleetEvents } from './build-fleet-events';
 import { buildMessageTrailEntry } from './build-message-trail-entry';
 import { buildReportTrailEntry } from './build-report-trail-entry';
+import { buildReportView } from './build-report-view';
 import { buildSessionEvent } from './build-session-event';
 import { buildSessionMessageEvent } from './build-session-message-event';
 import { buildSessionReportEvent } from './build-session-report-event';
@@ -51,6 +56,7 @@ import { IdempotencyLedger } from './idempotency-ledger';
 import { loadTranscriptPage } from './load-transcript-page';
 import { makeHookRunner } from './make-hook-runner';
 import type { HookScope } from './make-hook-runner';
+import { materializeWorkspace } from './materialize-workspace';
 import { mintMessageID } from './mint-message-id';
 import { mintSessionID } from './mint-session-id';
 import { parseReport } from './parse-report';
@@ -63,6 +69,7 @@ import { SessionManager } from './sessions';
 import type { Session, SessionDescriptor, SessionState } from './sessions';
 import { startEventsServer } from './start-events-server';
 import { startHeadlessTurn } from './start-headless-turn';
+import { startSessionBridge } from './start-session-bridge';
 import { TapRegistry } from './tap-registry';
 import type { TargetAccess } from './target-access';
 import { writeDaemonRecord } from './write-daemon-record';
@@ -129,9 +136,16 @@ export interface DaemonOptions {
   readonly restoreBootTimeoutMs?: number;
   readonly tapGraceMs?: number;
 
+  // How long a confirm token from `session.forget` stays usable.
+  readonly forgetConfirmMs?: number;
+
   // Called after a client-requested quit has stopped the daemon; the real
   // entrypoint exits the process, tests leave it unset.
   readonly onQuit?: () => void;
+
+  // Where background failures are reported, one line at a time; stderr
+  // when unset.
+  readonly log?: (line: string) => void;
 }
 
 export interface DaemonHandle {
@@ -145,6 +159,9 @@ export interface DaemonHandle {
 // it is refused.
 const TAP_GRACE_MS = 15_000;
 
+// How long a confirm token from a forget that destroys a host stays usable.
+const FORGET_CONFIRM_MS = 60_000;
+
 // The principal every request on the local socket acts as.
 const LOCAL_PRINCIPAL = 'local';
 
@@ -192,8 +209,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
   const store = await StateStore.open(opts.dbPath, opts.legacyFleetPath);
 
-  // Before any request is served: a key left in progress by a daemon that
-  // stopped mid-effect is settled first, so no retry can race it.
+  // Before any request is served: a workspace a stopped daemon left short
+  // of ready fails as interrupted, and a key left in progress by a daemon
+  // that stopped mid-effect is settled, so no retry can race either.
+  await store.reconcileMaterializations(Date.now());
   await store.reconcileIdempotencyKeys(Date.now());
 
   await tryRemoveExpiredIdempotencyKeys(store);
@@ -205,7 +224,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   idempotencySweep.unref();
 
   const targets =
-    opts.targets ?? buildExecutionTargets([{ id: 'local', provider: 'local-pty', options: {} }]);
+    opts.targets ??
+    buildExecutionTargets([{ id: 'local', provider: 'local-pty', options: {} }]).targets;
 
   const targetErrors = opts.targetErrors ?? [];
   const principals = opts.principals ?? null;
@@ -228,6 +248,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     targetErrors,
   );
 
+  if (opts.log !== undefined) {
+    mgr.log = opts.log;
+  }
+
   const clients = new Set<DaemonConnection>();
 
   const runHooks = makeHookRunner(opts.hooks ?? {});
@@ -265,14 +289,22 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   const eventSignal = new EventSignal();
 
   // A trail write that fails never fails the message request or report behind it.
-  const recordTrailEntry = async (entry: TrailEntry) => {
+  // Returns false when the entry was not written: a failed write, or a
+  // report the trail already holds.
+  const recordTrailEntry = async (entry: TrailEntry): Promise<boolean> => {
+    let written: boolean;
+
     try {
-      await store.recordTrailEntry(entry);
+      written = await store.recordTrailEntry(entry);
     } catch {
-      return;
+      return false;
     }
 
-    eventSignal.emit();
+    if (written) {
+      eventSignal.emit();
+    }
+
+    return written;
   };
 
   // Notes a message status change in the trail before broadcasting it, so a
@@ -419,7 +451,10 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     } catch {}
   };
 
-  const applyReport = async (e: HookEvent) => {
+  // A report id, which a remote session's reporter gives each report,
+  // makes a resent note land once; a resent answer changes nothing, since
+  // only an unanswered message takes one.
+  const applyReport = async (e: HookEvent, reportID?: string) => {
     const report = parseReport(e.payload);
 
     if (report === null) {
@@ -433,11 +468,22 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         const capped = { ...report, text: truncateToBytes(report.text, ANSWER_BYTE_CAP) };
         const reportedAt = Date.now();
 
-        await recordTrailEntry(
-          buildReportTrailEntry(sender.id, sender.agentSessionID, capped, reportedAt),
+        const entry = buildReportTrailEntry(
+          sender.id,
+          sender.agentSessionID,
+          capped,
+          reportedAt,
+          reportID,
         );
 
-        emitEvent(buildSessionReportEvent(sender.id, capped, reportedAt), findHookScope(sender.id));
+        const written = await recordTrailEntry(entry);
+
+        if (written) {
+          emitEvent(
+            buildSessionReportEvent(sender.id, capped, reportedAt),
+            findHookScope(sender.id),
+          );
+        }
       }
 
       return;
@@ -588,6 +634,20 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     }
   };
 
+  // A spawned terminal starts on a fresh screen; a revived one keeps the
+  // screen its session already had.
+  mgr.onBoot = (s, cols, rows) => {
+    const runtime = runtimes.get(s.id);
+
+    if (runtime === undefined) {
+      return;
+    }
+
+    runtime.resetBoot({ cols, rows });
+
+    runtime.screen ??= new ScreenModel(cols, rows);
+  };
+
   mgr.onOutput = (s, data) => {
     const runtime = runtimes.get(s.id);
 
@@ -713,7 +773,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     );
   };
 
-  const reporter = startHookServer((e) => {
+  const applyHookEvent = (e: HookEvent) => {
     if (e.event === 'Report') {
       void applyReport(e);
 
@@ -760,15 +820,53 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         void store.writeLastUsedAgent(started.agent);
       }
     }
-  }, opts.reporterSocketPath);
+  };
+
+  const reporter = startHookServer(applyHookEvent, opts.reporterSocketPath);
+
+  // A remote harness reports through a socket that serves it alone, so a
+  // line that names any other session is dropped.
+  mgr.onRelay = (binding, relay) => {
+    startSessionBridge(relay, binding, {
+      findSession: (sessionID) => mgr.sessions.find((x) => x.id === sessionID),
+      applyHookEvent,
+      applyReport: async (sessionID, payload, reportID) => {
+        if (parseReport(payload) === null) {
+          return false;
+        }
+
+        await applyReport({ atcId: sessionID, event: 'Report', payload: { ...payload } }, reportID);
+
+        return true;
+      },
+      attachTap: (client, sessionID) => ctx.attachTap(client, sessionID),
+      ackMessage: (client, sessionID, messageID) => ctx.ackMessage(client, sessionID, messageID),
+      detachTap: (client) => {
+        taps.detachAll(client);
+      },
+    });
+  };
+
+  // A spawn with a workspace source materializes it first, and the session
+  // registers only once its workspace is ready, so no session ever lists
+  // over a half-built checkout. A spawn that throws once its process has
+  // started takes the session back before it throws, so a failed start
+  // leaves nothing running and a keyed retry spawns once. When taking it
+  // back fails too, the session may still stand, and the throw says so.
+  const startSpawn = async (
+    p: SpawnParams,
+    id: SessionID,
+  ): Promise<Readonly<Record<string, unknown>>> => {
+    const prepared =
+      p.workspace === null ? null : await materializeSpawnWorkspace(p, id, p.workspace);
+
+    const materialized = prepared?.kind === 'ready' ? prepared : null;
+    const warnings = materialized === null ? [] : materialized.warnings;
 
-  // A spawn that throws once its process has started takes the session back
-  // before it throws, so a failed start leaves nothing running and a keyed
-  // retry spawns once. When taking it back fails too, the session may still
-  // stand, and the throw says so.
-  const startSpawn = async (p: SpawnParams, id: SessionID): Promise<SessionDescriptor> => {
     try {
-      return startSpawnedSession(p, id);
+      const session = await startSpawnedSession(p, id, materialized);
+
+      return warnings.length === 0 ? { session } : { session, warnings };
     } catch (error) {
       try {
         await mgr.removeFailedSpawn(id);
@@ -785,8 +883,45 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     }
   };
 
-  const startSpawnedSession = (p: SpawnParams, id: SessionID): SessionDescriptor => {
-    const s = mgr.spawn(
+  // Materializes a spawn's workspace on its target. The target's identity
+  // binds when the materialization starts, and every provider call after
+  // passes the execution gate against that binding.
+  const materializeSpawnWorkspace = (
+    p: SpawnParams,
+    id: SessionID,
+    source: SpawnWorkspaceSource,
+  ) => {
+    const bound = mgr.requireExecution({ target: p.target, targetIdentity: null }, 'run');
+    const binding = { target: p.target, targetIdentity: bound.identity };
+
+    return materializeWorkspace(
+      {
+        sessionID: id,
+        target: p.target,
+        dir: p.cwd,
+        source,
+        inPlace: bound.provider.kind === 'local-pty',
+      },
+      {
+        requireProvider: (capability) => mgr.requireExecution(binding, capability).provider,
+        store,
+        log: (line) => {
+          mgr.log(line);
+        },
+        stagingRoot: tmpdir(),
+      },
+    );
+  };
+
+  const startSpawnedSession = async (
+    p: SpawnParams,
+    id: SessionID,
+    materialized: Readonly<{
+      workspace: SessionWorkspace;
+      withheldEnv: readonly string[];
+    }> | null,
+  ): Promise<SessionDescriptor> => {
+    const s = await mgr.spawn(
       p.cwd,
       p.name,
       p.prompt,
@@ -799,15 +934,13 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       p.overrides,
       id,
       p.target,
+      materialized,
     );
 
     const runtime = runtimes.get(s.id);
 
     if (runtime !== undefined) {
       runtime.pendingLastUsed = true;
-      runtime.dims = { cols: p.cols, rows: p.rows };
-
-      runtime.screen = new ScreenModel(p.cols, p.rows);
     }
 
     void store.recordSpawnDir(p.cwd);
@@ -857,6 +990,49 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     }
   };
 
+  // Each confirm token a forget handed out, by token: the session it
+  // forgets, when it stops being taken, and whether a forget took it. A
+  // token stays known for one more lifetime after it expires, so a late
+  // forget learns it expired rather than that it never existed.
+  const confirmTokens = new Map<string, ConfirmToken>();
+
+  const forgetConfirmMs = opts.forgetConfirmMs ?? FORGET_CONFIRM_MS;
+
+  const claimConfirmToken = (sessionID: SessionID, token: string) => {
+    const now = Date.now();
+
+    for (const [held, entry] of confirmTokens) {
+      if (entry.expiresAt + forgetConfirmMs < now) {
+        confirmTokens.delete(held);
+      }
+    }
+
+    const entry = confirmTokens.get(token);
+    let reason: 'unknown' | 'used' | 'expired' | null = null;
+
+    if (entry === undefined || entry.session !== sessionID) {
+      reason = 'unknown';
+    } else if (entry.used) {
+      reason = 'used';
+    } else if (entry.expiresAt <= now) {
+      reason = 'expired';
+    }
+
+    if (reason !== null) {
+      throw new DaemonError(
+        'confirm_token_invalid',
+        `confirm token for session ${sessionID} is ${reason}; call session.forget without a token for a new one`,
+        { reason },
+      );
+    }
+
+    confirmTokens.set(token, {
+      session: sessionID,
+      expiresAt: entry?.expiresAt ?? now,
+      used: true,
+    });
+  };
+
   // Why the session refuses a message right now, or null when it takes one.
   const findMessageRefusal = (sessionID: SessionID): MessageRefusal | null => {
     const s = mgr.sessions.find((x) => x.id === sessionID);
@@ -952,6 +1128,18 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     );
   };
 
+  // Stops the headless runs a kill or forget ended, once the manager has
+  // acted: a session it removed already lost its run with its runtime, a
+  // session it left exited loses it here, and a sub-session it promoted, or
+  // any session of a kill or forget that failed, keeps working.
+  const stopEndedHeadlessRuns = (set: readonly Session[]) => {
+    for (const s of set) {
+      if (s.state === 'exited') {
+        runtimes.get(s.id)?.stopHeadlessRun();
+      }
+    }
+  };
+
   const ctx: DaemonContext = {
     build: opts.build,
     daemonID: store.daemonID,
@@ -996,9 +1184,13 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
       return target;
     },
+    requireWorkspaceTarget: (target) => {
+      mgr.requireExecution({ target, targetIdentity: null }, 'transfer');
+      mgr.requireExecution({ target, targetIdentity: null }, 'run');
+    },
     spawnSession: (plan, keyed, access) => {
       if (keyed === null) {
-        return startSpawn(plan(), mintSessionID()).then((session) => ({ session }));
+        return startSpawn(plan(), mintSessionID());
       }
 
       const effectRef = mintSessionID();
@@ -1007,7 +1199,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         operation: 'session.spawn',
         keyed,
         effectRef,
-        start: async () => ({ session: await startSpawn(plan(), effectRef) }),
+        start: () => startSpawn(plan(), effectRef),
         settle: () => mgr.writeFleet(),
         replay: (record) => {
           if (access !== null) {
@@ -1044,20 +1236,52 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
       for (const live of [s, ...mgr.collectChildren(id)]) {
         if (live.pty !== null) {
-          mgr.requireExecution(live, 'kill');
+          mgr.requireExecution(live, mgr.pickKillCapability(live));
         }
       }
 
-      for (const child of mgr.collectChildren(id)) {
-        runtimes.get(child.id)?.stopHeadlessRun();
-      }
-
-      runtimes.get(id)?.stopHeadlessRun();
+      const set = [s, ...mgr.collectChildren(id)];
 
       await mgr.kill(id);
 
+      stopEndedHeadlessRuns(set);
+
       return true;
     },
+
+    // A forget on a target that cannot destroy its host forgets at once. On
+    // one that can, a forget without a token checks the target and answers
+    // with a token, and the forget that carries the token destroys the host.
+    forgetSession: async (id, confirmToken) => {
+      const s = mgr.sessions.find((x) => x.id === id);
+
+      if (s === undefined) {
+        return 'missing';
+      }
+
+      if (mgr.findProvider(s)?.capabilities.destroy === true) {
+        mgr.requireExecution(s, 'destroy');
+
+        if (confirmToken === undefined) {
+          const token = randomUUID();
+          const expiresAt = Date.now() + forgetConfirmMs;
+
+          confirmTokens.set(token, { session: id, expiresAt, used: false });
+
+          return { confirmToken: token, expiresAt };
+        }
+
+        claimConfirmToken(id, confirmToken);
+      }
+
+      const set = [s, ...mgr.collectChildren(id)];
+
+      const destroyed = await mgr.forget(id);
+
+      stopEndedHeadlessRuns(set);
+
+      return { forgotten: true, destroyed };
+    },
     ejectSession: (id, prompt) => {
       const s = mgr.sessions.find((x) => x.id === id);
 
@@ -1101,7 +1325,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
       return 'ok';
     },
-    adoptSession: (id, cols, rows) => {
+    adoptSession: async (id, cols, rows) => {
       if (!hasResumableTranscript(mgr, id)) {
         return 'no_transcript';
       }
@@ -1109,18 +1333,13 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       const runtime = runtimes.get(id);
 
       runtime?.stopHeadlessRun();
-      const adopted = mgr.adoptTerminal(id, cols, rows);
+
+      const adopted = await mgr.adoptTerminal(id, cols, rows);
 
       if (adopted === null) {
         return 'missing';
       }
 
-      if (runtime !== undefined) {
-        runtime.resetBoot({ cols, rows });
-
-        runtime.screen ??= new ScreenModel(cols, rows);
-      }
-
       scheduleResize(id);
 
       return 'ok';
@@ -1233,6 +1452,30 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
       return 'ok';
     },
+    writeSessionLine: (sessionID, text) => {
+      const s = mgr.sessions.find((x) => x.id === sessionID);
+
+      // A headless turn takes the line as its prompt, and a missing or dead
+      // session refuses a line as it refuses raw input.
+      if (s === undefined || s.kind === 'headless' || s.pty === null) {
+        return ctx.writeSessionInput(sessionID, text);
+      }
+
+      mgr.requireExecution(s, 'input');
+
+      const bracketedPaste = runtimes.get(sessionID)?.screen?.hasBracketedPaste() ?? false;
+      const adapter = mgr.findAdapter(s.agent);
+      const writes = adapter?.planLineInput?.(text, { bracketedPaste }) ?? planTypedLineInput(text);
+
+      // Every write goes out in the tick the request arrives in, so no input
+      // overtakes the line and none lands between the text and its submit
+      // key.
+      for (const data of writes) {
+        s.pty.write(data);
+      }
+
+      return 'ok';
+    },
     resizeSession: (client, sessionID, dims) => {
       if (!attachments.updateDims(sessionID, client, dims)) {
         return false;
@@ -1327,6 +1570,11 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         await eventSignal.waitForNext(generation, remaining);
       }
     },
+    readReport: async (id, access) => {
+      const stored = await store.findReport(id, buildEventScope(null, access));
+
+      return stored === null ? null : buildReportView(stored, mgr.collectDescriptors());
+    },
     writeSessionMessage: async (sessionID, from, text, keyed) => {
       if (keyed === null) {
         const refusal = findMessageRefusal(sessionID);
@@ -1493,7 +1741,11 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     server.stop(true);
     eventsServer?.stop();
     reporter.stop(true);
-    mgr.killAll();
+    mgr.detachAll();
+
+    for (const target of targets) {
+      target.provider?.dispose();
+    }
 
     for (const runtime of runtimes.values()) {
       runtime.dispose();
@@ -1557,6 +1809,12 @@ function hasResumableTranscript(mgr: SessionManager, id: SessionID): boolean {
     return false;
   }
 
+  // A remote session's transcript lives in its host, out of the daemon's
+  // reach, so the agent's session id alone makes it resumable.
+  if (mgr.findProvider(s)?.remote === true) {
+    return s.agentSessionID !== undefined;
+  }
+
   return adapter.canResume({
     ...(s.agentSessionID === undefined ? {} : { agentSessionID: s.agentSessionID }),
     ...(s.transcriptSource === undefined ? {} : { transcriptSource: s.transcriptSource }),
@@ -1584,6 +1842,12 @@ async function tryRemoveExpiredIdempotencyKeys(store: StateStore): Promise<boole
   return true;
 }
 
+interface ConfirmToken {
+  readonly session: SessionID;
+  readonly expiresAt: number;
+  readonly used: boolean;
+}
+
 // Carries a message refusal out of a keyed send's start, so the claim drops
 // and the refusal still answers the request.
 class MessageRefusedError extends Error {