@zgeoff/atc 2.11.0 → 2.13.0

package/src/daemon/effect-remains-error.ts

@@ -0,0 +1,13 @@
+/**
+ * Thrown by an idempotent effect's start when it failed after the effect
+ * began and could not confirm taking it back, so the effect may still stand.
+ * The ledger keeps the key as outcome_unknown instead of releasing it, and a
+ * retry under the key never runs the effect a second time.
+ */
+export class EffectRemainsError extends Error {
+  constructor(message: string, options?: Readonly<ErrorOptions>) {
+    super(message, options);
+
+    this.name = 'EffectRemainsError';
+  }
+}

package/src/daemon/execution-provider.ts

@@ -0,0 +1,208 @@
+/**
+ * The host a session's harness runs on: it starts a process in a
+ * pseudo-terminal and, as its capabilities declare, unpacks files into its
+ * filesystem and runs commands there. The daemon starts every session
+ * terminal through this interface, so a new host is a provider, not a
+ * change to the session code. The daemon checks a capability before it
+ * calls the operation behind it, and a request needing one the provider
+ * lacks fails with `unsupported_operation`.
+ */
+export interface ExecutionProvider {
+  // The provider kind a target's config selects, such as `local-pty`.
+  readonly kind: string;
+  readonly capabilities: ExecutionCapabilities;
+
+  // Whether the host is a machine other than the daemon's own. A remote
+  // harness's files, transcripts included, live on that machine, and its
+  // environment holds only what atc sets for it.
+  readonly remote: boolean;
+
+  // Where a remote host keeps atc's files: the folder each session's own
+  // files unpack under, and the atc binary inside the host, null when the
+  // host has none. Absent on the daemon's own machine.
+  readonly guest?: GuestLayout;
+
+  // Readies the host a harness is about to start on: a remote host is
+  // created when missing, woken when asleep, and held awake while its
+  // harnesses run. Rejects with the refusal before any harness starts.
+  readonly prepareHost: (request: HostRequest) => Promise<void>;
+
+  // Starts a process in a pseudo-terminal of the given size, on the host the
+  // spec holds, which a prepare readied first.
+  readonly spawnHarness: (spec: HarnessSpec) => HarnessHandle;
+
+  // Unpacks a tar archive into a directory on a host, creating the directory
+  // first. The daemon's own machine takes no host.
+  // oxlint-disable-next-line prefer-readonly-parameter-types -- archive bytes have no readonly form
+  readonly transferArchive: (archive: Uint8Array, dir: string, host?: string) => Promise<void>;
+
+  // Runs a command to completion and returns what it printed.
+  readonly runCommand: (spec: CommandSpec) => Promise<CommandResult>;
+
+  // Puts a host to sleep with every harness on it kept inside, so a revive
+  // finds each one as it was. Rejects with `host_leased` when another owner
+  // keeps the host awake, and leaves the host as it was then.
+  readonly suspendHost: (host: string) => Promise<void>;
+
+  // Deletes a host and everything on it, harnesses included. Nothing brings
+  // a destroyed host back.
+  readonly destroyHost: (host: string) => Promise<void>;
+
+  // Stops the provider's own background work when the daemon stops, and
+  // leaves every remote harness running for the next daemon to find.
+  readonly dispose: () => void;
+}
+
+/**
+ * The host a harness is about to start on, and the daemon that holds it:
+ * a remote host is held per daemon, so two daemons never release each
+ * other's hold.
+ */
+export interface HostRequest {
+  readonly host: string;
+  readonly daemonID: string;
+
+  // Whether the harness about to start needs atc inside the host, which a
+  // provider that ships its own binary installs when missing.
+  readonly installATC?: boolean;
+}
+
+export interface GuestLayout {
+  readonly dir: string;
+  readonly atc: string | null;
+}
+
+/**
+ * What a provider can do. The terminal capabilities split so a host that can
+ * show output but not take input, or not resize, says so; `suspend` and
+ * `destroy` cover the lifecycle of a host that outlives its process.
+ */
+export interface ExecutionCapabilities {
+  // Start a harness in a pseudo-terminal.
+  readonly spawn: boolean;
+
+  // Stream a running harness's output to attached clients.
+  readonly attach: boolean;
+
+  // Write keystrokes to a running harness.
+  readonly input: boolean;
+
+  // Change a running harness's terminal size.
+  readonly resize: boolean;
+
+  // End a running harness.
+  readonly kill: boolean;
+
+  // Unpack an archive into the host's filesystem.
+  readonly transfer: boolean;
+
+  // Run a command on the host to completion.
+  readonly run: boolean;
+
+  // Run an agent turn without a terminal, through the agent's own runner.
+  readonly headless: boolean;
+
+  // Pause the host and resume it later with its state intact.
+  readonly suspend: boolean;
+
+  // Delete the host and everything on it.
+  readonly destroy: boolean;
+}
+
+export type ExecutionCapability = keyof ExecutionCapabilities;
+
+export interface HarnessSpec {
+  // The atc session the harness belongs to, and the host it runs on: the
+  // session's own id, or its parent's when the two share a host.
+  readonly session: string;
+  readonly host: string;
+  readonly bin: string;
+  readonly args: readonly string[];
+  readonly cwd: string;
+
+  // The variables atc sets for the harness. A provider for the daemon's
+  // own machine adds the daemon's environment around them; a remote one
+  // passes only these and its own.
+  readonly env: Readonly<Record<string, string>>;
+
+  // The variable names the harness goes without even when the daemon's
+  // environment holds them, such as a workspace's credential; none of the
+  // variables atc sets is withheld.
+  readonly withheldEnv?: readonly string[];
+  readonly cols: number;
+  readonly rows: number;
+
+  // Takes each connection a process of the harness opens to the daemon. A
+  // remote provider relays them from a socket inside the host that serves
+  // this harness alone, and points the harness's ATC_SOCKET at it.
+  readonly onRelay?: (relay: HarnessRelay) => void;
+}
+
+/**
+ * One connection from a process inside a harness's host to the daemon, in
+ * lines: each line the process writes arrives whole, and each line the
+ * daemon writes reaches the process in order.
+ */
+export interface HarnessRelay {
+  readonly onLine: (listener: (line: string) => void) => void;
+  readonly onClose: (listener: () => void) => void;
+
+  // Settles once the relay has room for more.
+  readonly writeLine: (line: string) => Promise<void>;
+  readonly close: () => void;
+}
+
+/**
+ * A running harness. Output and exit arrive through listeners, each
+ * subscription detachable on its own.
+ */
+export interface HarnessHandle {
+  readonly onData: (listener: (data: string) => void) => HarnessSubscription;
+  readonly onExit: (listener: (exit: HarnessExit) => void) => HarnessSubscription;
+
+  // Follows a remote harness's connection: lost and being restored, or
+  // restored. A harness on the daemon's own machine has no connection.
+  readonly onAttachment?: (
+    listener: (attachment: HarnessAttachment) => void,
+  ) => HarnessSubscription;
+  readonly write: (data: string) => void;
+  readonly resize: (cols: number, rows: number) => void;
+  readonly kill: () => void;
+
+  // Stops following the harness and leaves its process running, for a host
+  // that keeps it after the daemon lets go; on a host that cannot keep it,
+  // the process ends as a kill ends it. No listener fires after a detach.
+  readonly detach: () => void;
+}
+
+export type HarnessAttachment = 'attached' | 'reattaching';
+
+interface HarnessSubscription {
+  readonly dispose: () => void;
+}
+
+export interface HarnessExit {
+  readonly exitCode: number;
+
+  // Why the harness stopped: its process exited, its host went to sleep
+  // with the process inside, or the host lost it without an exit, such as
+  // a cold boot. Absent is an exit.
+  readonly reason?: 'exited' | 'suspended' | 'ended';
+
+  // What ended it, for a reason other than an exit.
+  readonly detail?: string;
+}
+
+export interface CommandSpec {
+  readonly argv: readonly string[];
+  readonly cwd: string;
+
+  // The host the command runs on; the daemon's own machine takes none.
+  readonly host?: string;
+}
+
+export interface CommandResult {
+  readonly exitCode: number;
+  readonly stdout: string;
+  readonly stderr: string;
+}

package/src/daemon/find-execution-refusal.ts

@@ -0,0 +1,104 @@
+import { DaemonError } from '../protocol/daemon-error';
+import type { TargetConfigError } from '../shared/collect-targets';
+import type { ExecutionTarget } from './build-execution-targets';
+import type { ExecutionCapability } from './execution-provider';
+
+// The target a session runs on and the identity it was bound to there; a
+// null identity is a session not yet bound, which binds to the target as
+// it stands now. A null target is a spawn that names none when the config
+// gives no default.
+interface TargetBinding {
+  readonly target: string | null;
+  readonly targetIdentity: string | null;
+}
+
+/**
+ * The refusal for running work of the given capability on a session's
+ * target, or null when the target serves it. Every path that starts a
+ * harness, writes to one, or runs a headless turn asks this first, so no
+ * path runs a session anywhere but the target it is bound to:
+ *
+ * - `target_config_invalid` when the config file exists but cannot be
+ *   read or parsed, which refuses every target, `local` included; when the
+ *   config's `targets` map, or the target's own entry, is malformed; and
+ *   when there is no target because the config gives no default.
+ * - `unknown_target` when no target holds the id.
+ * - `target_changed` when the target's identity is not the one the session
+ *   was bound to: the name now holds another provider or other options.
+ * - `target_unavailable` when this daemon has no provider of its kind.
+ * - `unsupported_operation` when the provider lacks the capability.
+ */
+export function findExecutionRefusal(
+  targets: ReadonlyMap<string, ExecutionTarget>,
+  errors: readonly TargetConfigError[],
+  binding: TargetBinding,
+  capability: ExecutionCapability,
+): DaemonError | null {
+  const fileError = errors.find((error) => error.scope === 'config');
+
+  if (fileError !== undefined) {
+    return new DaemonError(
+      'target_config_invalid',
+      `config file ${fileError.path} cannot be used (${fileError.problem}: ${fileError.detail}), so no session runs on any target, local included. Fix the file and restart the daemon`,
+      { problem: fileError.problem, path: fileError.path, detail: fileError.detail },
+    );
+  }
+
+  const id = binding.target;
+
+  if (id === null) {
+    const problem =
+      errors.find((error) => error.scope !== 'target')?.problem ??
+      'the targets map holds no local target and no defaultTarget is set';
+
+    return new DaemonError(
+      'target_config_invalid',
+      `no default execution target: ${problem}. Name a target on the spawn, or fix targets in config.json and restart the daemon`,
+      { problem },
+    );
+  }
+
+  const configError = errors.find(
+    (error) => error.scope === 'targets' || (error.scope === 'target' && error.target === id),
+  );
+
+  if (configError !== undefined) {
+    return new DaemonError(
+      'target_config_invalid',
+      `execution target '${id}' cannot be used: ${configError.problem}. Fix targets in config.json and restart the daemon`,
+      { target: id, problem: configError.problem },
+    );
+  }
+
+  const target = targets.get(id);
+
+  if (target === undefined) {
+    return new DaemonError('unknown_target', `no execution target '${id}'`, { target: id });
+  }
+
+  if (binding.targetIdentity !== null && binding.targetIdentity !== target.identity) {
+    return new DaemonError(
+      'target_changed',
+      `execution target '${id}' changed since this session started on it (was ${binding.targetIdentity}, now ${target.identity}). Restore the target's earlier config to use the session, or kill it`,
+      { target: id, boundIdentity: binding.targetIdentity, currentIdentity: target.identity },
+    );
+  }
+
+  if (target.provider === null) {
+    return new DaemonError(
+      'target_unavailable',
+      `execution target '${id}' needs a '${target.kind}' provider, which this daemon does not have`,
+      { target: id, provider: target.kind },
+    );
+  }
+
+  if (!target.provider.capabilities[capability]) {
+    return new DaemonError(
+      'unsupported_operation',
+      `the ${target.provider.kind} execution provider does not support ${capability}`,
+      { provider: target.provider.kind, capability },
+    );
+  }
+
+  return null;
+}

package/src/daemon/hooks.ts

@@ -1,8 +1,7 @@
 import { unlinkSync } from 'node:fs';
 import { socketPath } from '../shared/config';
-import { isRecord } from '../shared/report';
 import type { SessionID } from '../shared/session-id';
-import { toSessionID } from '../shared/to-session-id';
+import { parseHookLine } from './parse-hook-line';
 
 export interface HookEvent {
   atcId: SessionID;
@@ -38,22 +37,11 @@ export function startHookServer(onEvent: (e: HookEvent) => void, path: string =
             continue;
           }
 
-          try {
-            const parsed: unknown = JSON.parse(line);
+          const event = parseHookLine(line);
 
-            if (
-              isRecord(parsed) &&
-              typeof parsed['atcId'] === 'string' &&
-              typeof parsed['event'] === 'string' &&
-              isRecord(parsed['payload'])
-            ) {
-              onEvent({
-                atcId: toSessionID(parsed['atcId']),
-                event: parsed['event'],
-                payload: parsed['payload'],
-              });
-            }
-          } catch {}
+          if (event !== null) {
+            onEvent(event);
+          }
         }
       },
       open(socket) {

package/src/daemon/idempotency-ledger.ts

@@ -0,0 +1,164 @@
+import { DaemonError } from '../protocol/daemon-error';
+import type { EffectTarget, IdempotencyRecord } from '../store/idempotency-record';
+import type { StateStore } from '../store/state-store';
+import { EffectRemainsError } from './effect-remains-error';
+
+// A request's idempotency key, the hash of the payload it came with, and
+// the principal the request acts as when it is not the ledger's own.
+export interface KeyedRequest {
+  readonly key: string;
+  readonly payloadHash: string;
+  readonly principal?: string;
+}
+
+interface IdempotentCall<T> {
+  readonly operation: string;
+  readonly keyed: KeyedRequest;
+
+  // The id the effect runs under, minted before the claim records it.
+  readonly effectRef: string;
+
+  // Starts the effect. A plain throw means nothing took effect, so the claim
+  // is dropped for a retry to run fresh; an EffectRemainsError means the
+  // effect may still stand, so the claim is kept as outcome_unknown.
+  readonly start: () => T | Promise<T>;
+
+  // Resolves once the effect is durable; the claim completes only after.
+  readonly settle: () => Promise<void>;
+
+  // The answer to a retry of a completed key.
+  readonly replay: (record: IdempotencyRecord) => T | Promise<T>;
+
+  // The target the effect's session is bound to, recorded with the
+  // completed key; absent or null for an effect bound to none.
+  readonly findEffectTarget?: (result: T) => EffectTarget | null;
+}
+
+/**
+ * Runs keyed effects at most once per key for each principal: the
+ * request's own, else the ledger's. The first
+ * request claims the key with its payload's hash and the pre-minted effect
+ * id, runs the effect, and completes the key with its answer once the
+ * effect is durable. A retry with the same payload replays the completed
+ * answer; one with a different payload is `idempotency_conflict`; one whose
+ * effect a stopped daemon may or may not have run is `outcome_unknown` with
+ * the effect id in `data.effectRef`, and never starts the effect again.
+ */
+export class IdempotencyLedger {
+  private readonly store: StateStore;
+
+  private readonly principal: string;
+
+  // One promise chain per key, so two requests under the same key never
+  // interleave their claim and completion on one daemon.
+  private readonly locks = new Map<string, Promise<void>>();
+
+  constructor(store: StateStore, principal: string) {
+    this.store = store;
+    this.principal = principal;
+  }
+
+  async run<T extends Readonly<Record<string, unknown>>>(call: IdempotentCall<T>): Promise<T> {
+    const lockKey = JSON.stringify([
+      call.keyed.principal ?? this.principal,
+      call.operation,
+      call.keyed.key,
+    ]);
+
+    const previous = this.locks.get(lockKey) ?? Promise.resolve();
+    const released = Promise.withResolvers<void>();
+
+    const chained = (async () => {
+      await previous;
+      await released.promise;
+    })();
+
+    this.locks.set(lockKey, chained);
+
+    await previous;
+
+    try {
+      return await this.runClaimed(call);
+    } finally {
+      released.resolve();
+
+      if (this.locks.get(lockKey) === chained) {
+        this.locks.delete(lockKey);
+      }
+    }
+  }
+
+  private async runClaimed<T extends Readonly<Record<string, unknown>>>(
+    call: IdempotentCall<T>,
+  ): Promise<T> {
+    const id = {
+      principal: call.keyed.principal ?? this.principal,
+      operation: call.operation,
+      key: call.keyed.key,
+    };
+
+    const held = await this.store.claimIdempotencyKey({
+      ...id,
+      payloadHash: call.keyed.payloadHash,
+      effectRef: call.effectRef,
+      at: Date.now(),
+    });
+
+    if (held !== null) {
+      return answerHeldKey(call, held);
+    }
+
+    let result: T;
+
+    try {
+      result = await call.start();
+    } catch (error) {
+      if (error instanceof EffectRemainsError) {
+        await this.store.updateIdempotencyOutcomeUnknown(id, Date.now());
+
+        throw new DaemonError(
+          'outcome_unknown',
+          `the ${call.operation} under idempotency key '${call.keyed.key}' failed and its effect may still stand; check ${call.effectRef} before retrying under a new key`,
+          { effectRef: call.effectRef },
+        );
+      }
+
+      await this.store.removeIdempotencyKey(id);
+
+      throw error;
+    }
+
+    // A settle that fails leaves the claim in progress: the effect started,
+    // so the next daemon start marks its outcome unknown rather than letting
+    // a retry run it again.
+    await call.settle();
+
+    await this.store.updateIdempotencyCompleted(
+      id,
+      JSON.stringify(result),
+      Date.now(),
+      call.findEffectTarget?.(result) ?? null,
+    );
+
+    return result;
+  }
+}
+
+function answerHeldKey<T>(call: IdempotentCall<T>, held: IdempotencyRecord): T | Promise<T> {
+  if (held.payloadHash !== call.keyed.payloadHash) {
+    throw new DaemonError(
+      'idempotency_conflict',
+      `idempotency key '${held.key}' was first used with a different ${held.operation} payload`,
+    );
+  }
+
+  if (held.state === 'completed') {
+    return call.replay(held);
+  }
+
+  throw new DaemonError(
+    'outcome_unknown',
+    `the ${held.operation} under idempotency key '${held.key}' was interrupted and may or may not have taken effect; check ${held.effectRef} before retrying under a new key`,
+    { effectRef: held.effectRef },
+  );
+}