@zgeoff/atc 2.20.0 → 2.23.0

package/src/federation/gateway-store.ts

@@ -0,0 +1,253 @@
+import { Database } from 'bun:sqlite';
+import { DaemonError } from '../protocol/daemon-error';
+
+/**
+ * The last outcome the gateway saw for a keyed request: `pending` until an
+ * answer arrives, `completed` for any answer the daemon gave other than
+ * `outcome_unknown`, and `uncertain` for `outcome_unknown` or a response
+ * that never arrived.
+ */
+export type BindingOutcome = 'pending' | 'completed' | 'uncertain';
+
+/**
+ * The daemon a keyed spawn or message went to: bound per principal,
+ * operation, and key before the request leaves the gateway, so every retry
+ * reaches the same daemon whatever the default daemon is by then. The
+ * daemon's announced completed-key retention, null when it announced none,
+ * decides when the binding may go. The payload hash binds the key to the
+ * request it was first used with, as the daemon's own ledger does.
+ */
+export interface KeyBinding {
+  readonly principal: string;
+  readonly operation: string;
+  readonly key: string;
+  readonly daemon: string;
+  readonly daemonID: string;
+  readonly retentionMs: number | null;
+  readonly payloadHash: string;
+  readonly outcome: BindingOutcome;
+  readonly outcomeAt: number;
+
+  // When a request under the key first left for its daemon, null while
+  // none has, and the daemon's id of the effect an uncertain answer
+  // returned, null without one.
+  readonly sentAt: number | null;
+  readonly effectRef: string | null;
+
+  // The id of the claim that wrote the binding, so a call can tell its own
+  // binding from one another call wrote first.
+  readonly claimID: string;
+}
+
+interface BindingRow {
+  readonly principal: string;
+  readonly operation: string;
+  readonly key: string;
+  readonly daemon: string;
+  readonly daemon_id: string;
+  readonly retention_ms: number | null;
+  readonly payload_hash: string;
+  readonly outcome: string;
+  readonly outcome_at: number;
+  readonly sent_at: number | null;
+  readonly effect_ref: string | null;
+  readonly claim_id: string;
+}
+
+/**
+ * `gateway.db`: the keyed-request bindings, and nothing else the gateway
+ * could rebuild. A binding outlives the daemon's promise to deduplicate its
+ * key: a completed binding goes only once twice the daemon's announced
+ * retention has passed since its answer, and a pending or uncertain one,
+ * or one bound to a daemon that announced no retention, stays for good, as
+ * the daemon keeps such a key.
+ */
+export class GatewayStore {
+  private readonly db: Database;
+
+  private constructor(db: Database) {
+    this.db = db;
+  }
+
+  static open(path: string): GatewayStore {
+    const db = new Database(path, { create: true, strict: true });
+
+    db.run('PRAGMA journal_mode = WAL');
+
+    db.run(`CREATE TABLE IF NOT EXISTS key_binding (
+      principal TEXT NOT NULL,
+      operation TEXT NOT NULL,
+      key TEXT NOT NULL,
+      daemon TEXT NOT NULL,
+      daemon_id TEXT NOT NULL,
+      retention_ms INTEGER,
+      payload_hash TEXT NOT NULL,
+      outcome TEXT NOT NULL,
+      outcome_at INTEGER NOT NULL,
+      sent_at INTEGER,
+      effect_ref TEXT,
+      claim_id TEXT NOT NULL,
+      PRIMARY KEY (principal, operation, key)
+    )`);
+
+    return new GatewayStore(db);
+  }
+
+  /**
+   * Binds the key to the daemon under the given claim id unless a binding
+   * for it already exists, and
+   * returns the binding that holds after the call: the new one, or the one
+   * an earlier request made, whose daemon the request must go to. Throws
+   * `idempotency_conflict` when the key's binding holds another payload, so
+   * a reused key never reaches a daemon that may have dropped it already.
+   */
+  claimBinding(
+    binding: Omit<KeyBinding, 'outcome' | 'outcomeAt' | 'sentAt' | 'effectRef'>,
+    now: number,
+  ): KeyBinding {
+    this.db
+      .query(
+        `INSERT INTO key_binding
+           (principal, operation, key, daemon, daemon_id, retention_ms, payload_hash, outcome, outcome_at, claim_id)
+         VALUES ($principal, $operation, $key, $daemon, $daemonID, $retentionMs, $payloadHash, 'pending', $now, $claimID)
+         ON CONFLICT (principal, operation, key) DO NOTHING`,
+      )
+      .run({
+        principal: binding.principal,
+        operation: binding.operation,
+        key: binding.key,
+        daemon: binding.daemon,
+        daemonID: binding.daemonID,
+        retentionMs: binding.retentionMs,
+        payloadHash: binding.payloadHash,
+        claimID: binding.claimID,
+        now,
+      });
+
+    const held = this.findBinding(binding.principal, binding.operation, binding.key);
+
+    if (held === null) {
+      throw new Error('a claimed key binding is missing');
+    }
+
+    if (held.payloadHash !== binding.payloadHash) {
+      throw new DaemonError(
+        'idempotency_conflict',
+        `idempotency key '${binding.key}' was first used with a different ${binding.operation} payload`,
+      );
+    }
+
+    return held;
+  }
+
+  findBinding(principal: string, operation: string, key: string): KeyBinding | null {
+    const row = this.db
+      .query<BindingRow, { principal: string; operation: string; key: string }>(
+        `SELECT * FROM key_binding
+         WHERE principal = $principal AND operation = $operation AND key = $key`,
+      )
+      .get({ principal, operation, key });
+
+    return row === null ? null : toKeyBinding(row);
+  }
+
+  /**
+   * Takes the key's first send: marks the binding sent unless a request
+   * under the key was sent before, in one statement, so of any number of
+   * calls, across gateway processes too, exactly one takes it. Returns
+   * whether this call took it: only that call's request may run the
+   * effect, and every later one may only replay it.
+   */
+  claimFirstSend(principal: string, operation: string, key: string, now: number): boolean {
+    return (
+      this.db
+        .query(
+          `UPDATE key_binding SET sent_at = $now
+           WHERE principal = $principal AND operation = $operation AND key = $key
+             AND sent_at IS NULL`,
+        )
+        .run({ principal, operation, key, now }).changes === 1
+    );
+  }
+
+  /**
+   * Records the last outcome of the key's request, and the effect id an
+   * uncertain answer returned, keeping one recorded earlier when this
+   * answer holds none. A completed outcome stays completed: a later
+   * request under the key whose answer never arrived changes nothing the
+   * daemon already answered.
+   */
+  updateOutcome(
+    principal: string,
+    operation: string,
+    key: string,
+    outcome: BindingOutcome,
+    now: number,
+    effectRef: string | null = null,
+  ): void {
+    this.db
+      .query(
+        `UPDATE key_binding
+         SET outcome = $outcome, outcome_at = $now, effect_ref = COALESCE($effectRef, effect_ref)
+         WHERE principal = $principal AND operation = $operation AND key = $key
+           AND (outcome != 'completed' OR $outcome = 'completed')`,
+      )
+      .run({ principal, operation, key, outcome, now, effectRef });
+  }
+
+  /**
+   * Removes the key's binding when the given claim wrote it and no request
+   * under the key was sent, for a request the gateway refused before it
+   * sent anything. A binding another claim wrote, or one sent, stays.
+   */
+  removeBinding(principal: string, operation: string, key: string, claimID: string): void {
+    this.db
+      .query(
+        `DELETE FROM key_binding
+         WHERE principal = $principal AND operation = $operation AND key = $key
+           AND claim_id = $claimID AND sent_at IS NULL`,
+      )
+      .run({ principal, operation, key, claimID });
+  }
+
+  /**
+   * Removes every completed binding whose daemon announced a retention and
+   * whose answer is older than twice that retention. Returns how many went.
+   */
+  removeExpiredBindings(now: number): number {
+    return this.db
+      .query(
+        `DELETE FROM key_binding
+         WHERE outcome = 'completed' AND retention_ms IS NOT NULL
+           AND outcome_at + 2 * retention_ms < $now`,
+      )
+      .run({ now }).changes;
+  }
+
+  stop(): void {
+    this.db.close();
+  }
+}
+
+function toKeyBinding(row: BindingRow): KeyBinding {
+  return {
+    principal: row.principal,
+    operation: row.operation,
+    key: row.key,
+    daemon: row.daemon,
+    daemonID: row.daemon_id,
+    retentionMs: row.retention_ms,
+    payloadHash: row.payload_hash,
+    outcome: pickOutcome(row.outcome),
+    outcomeAt: row.outcome_at,
+    sentAt: row.sent_at,
+    effectRef: row.effect_ref,
+    claimID: row.claim_id,
+  };
+}
+
+// A stored outcome; anything unreadable counts as uncertain, which keeps
+// the binding.
+function pickOutcome(value: string): BindingOutcome {
+  return value === 'pending' || value === 'completed' ? value : 'uncertain';
+}

package/src/federation/id-rules.ts

@@ -0,0 +1,85 @@
+/**
+ * How the gateway treats one field of a daemon answer: `id` prefixes a
+ * daemon id with the daemon's name and incarnation, `locator` replaces the
+ * daemon ID in a session locator with the name and incarnation, `cursor`
+ * is a daemon cursor the event merge replaces with a gateway cursor, and
+ * `keep` passes a value that looks like an id but is none of atc's, such as
+ * an agent's own session id, while still rewriting ruled fields below it.
+ * `opaque` passes a whole value unread: text an agent or a terminal wrote,
+ * or a cursor only the same daemon reads back.
+ */
+export type IDRule = 'id' | 'locator' | 'cursor' | 'keep' | 'opaque';
+
+// The fields of a session descriptor, relative to the descriptor.
+const DESCRIPTOR_RULES: readonly (readonly [string, IDRule])[] = [
+  ['id', 'id'],
+  ['parent', 'id'],
+  ['children[]', 'id'],
+  ['locator', 'locator'],
+  ['agentSessionID', 'keep'],
+];
+
+/**
+ * The rule for every id-bearing field of each daemon method's answer the
+ * gateway passes on, by the field's path: object keys joined with `.`, and
+ * `[]` for every element of an array. An empty map is a method whose answer
+ * holds no id; a method missing here is one the gateway does not pass on.
+ */
+export const ID_RULES: Readonly<Record<string, ReadonlyMap<string, IDRule>>> = {
+  'agents.list': new Map(),
+  'dirs.list': new Map([['dirs', 'opaque']]),
+  'session.read': new Map([
+    ['rows', 'opaque'],
+    ['cursor', 'opaque'],
+  ]),
+  'session.screen': new Map([['text', 'opaque']]),
+  'session.resumeCommand': new Map([['command', 'opaque']]),
+  'session.submit': new Map(),
+  'session.update': new Map(),
+  'session.kill': new Map(),
+  'session.ack': new Map(),
+  'session.list': buildPrefixedRules('sessions[].', DESCRIPTOR_RULES),
+  'session.spawn': buildPrefixedRules('session.', DESCRIPTOR_RULES),
+  'session.get': buildPrefixedRules('session.', DESCRIPTOR_RULES),
+  'session.message': new Map([['message', 'id']]),
+  'message.get': new Map([
+    ['message', 'id'],
+    ['session', 'id'],
+    ['answeredWith[]', 'id'],
+    ['turn', 'keep'],
+    ['turn.session', 'id'],
+  ]),
+  'message.ack': new Map([['message', 'id']]),
+  'report.get': new Map([
+    ['report', 'id'],
+    ['session', 'id'],
+    ['text', 'opaque'],
+  ]),
+  'events.read': new Map([
+    ['events[].session', 'id'],
+    ['events[].message', 'id'],
+    ['events[].parent', 'id'],
+    ['events[].report', 'id'],
+    ['events[].cursor', 'cursor'],
+    ['cursor', 'cursor'],
+  ]),
+};
+
+/**
+ * The rule for every id-bearing field of an error's `data`, whatever the
+ * method: `effectRef` holds the session or message an uncertain or
+ * conflicting keyed request made.
+ */
+export const ERROR_DATA_RULES: ReadonlyMap<string, IDRule> = new Map([
+  ['effectRef', 'id'],
+  ['session', 'id'],
+  ['message', 'id'],
+  ['parent', 'id'],
+]);
+
+function buildPrefixedRules(
+  prefix: string,
+  rules: readonly (readonly [string, IDRule])[],
+): ReadonlyMap<string, IDRule> {
+  return new Map(rules.map(([path, rule]) => [`${prefix}${path}`, rule]));
+}

package/src/federation/load-gateway-registry.ts

@@ -0,0 +1,29 @@
+import { readFileSync } from 'node:fs';
+import { parseGatewayRegistry } from './parse-gateway-registry';
+import type { GatewayRegistry } from './types';
+
+type LoadedGatewayRegistry =
+  | { readonly ok: true; readonly registry: GatewayRegistry }
+  | { readonly ok: false; readonly errors: readonly string[] };
+
+/**
+ * Reads the registry file and parses it with the tokens from the
+ * environment. An unreadable file or one that is not JSON refuses the
+ * registry like any other problem in it.
+ */
+export function loadGatewayRegistry(
+  path: string,
+  env: Readonly<Record<string, string | undefined>>,
+): LoadedGatewayRegistry {
+  let raw: unknown;
+
+  try {
+    raw = JSON.parse(readFileSync(path, 'utf8'));
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+
+    return { ok: false, errors: [`cannot read the registry at ${path}: ${detail}`] };
+  }
+
+  return parseGatewayRegistry(raw, env);
+}

package/src/federation/max-events-cursor-bytes.ts

@@ -0,0 +1,4 @@
+/**
+ * The largest gateway events cursor the gateway reads back, in bytes.
+ */
+export const MAX_EVENTS_CURSOR_BYTES = 4096;

package/src/federation/max-registry-daemons.ts

@@ -0,0 +1,26 @@
+import { encodeCursor } from '../protocol/encode-cursor';
+import { encodeGatewayCursor } from './encode-gateway-cursor';
+import { MAX_EVENTS_CURSOR_BYTES } from './max-events-cursor-bytes';
+
+/**
+ * The most daemons a registry may list: the largest count whose worst-case
+ * events cursor still fits the bytes the gateway reads back. The worst case
+ * gives every daemon a name of the longest allowed length and a position at
+ * the largest event id a daemon cursor can hold, under the longest filter
+ * hash, so any cursor a registry this size produces decodes again.
+ */
+export const MAX_REGISTRY_DAEMONS = (() => {
+  const position = encodeCursor({ kind: 'events', id: Number.MAX_SAFE_INTEGER });
+
+  const parts = new Map<string, string>();
+
+  for (;;) {
+    const index = String(parts.size);
+
+    parts.set(`${'a'.repeat(31 - index.length)}${index}.ffffffff`, position);
+
+    if (Buffer.byteLength(encodeGatewayCursor('f'.repeat(22), parts)) > MAX_EVENTS_CURSOR_BYTES) {
+      return parts.size - 1;
+    }
+  }
+})();

package/src/federation/merge-event-pages.ts

@@ -0,0 +1,228 @@
+import { decodeCursor } from '../protocol/decode-cursor';
+import { encodeCursor } from '../protocol/encode-cursor';
+import { isRecord } from '../shared/report';
+import { buildGatewayID } from './build-gateway-id';
+import { buildRuledValue } from './build-ruled-value';
+import { encodeGatewayCursor } from './encode-gateway-cursor';
+import type { IDRule } from './id-rules';
+import type { RegistryDaemon } from './types';
+
+/**
+ * What one daemon gave an events read: a page of its events in its own
+ * order, with the cursor it returned and whether more follow; nothing,
+ * because it did not answer; or, for a daemon that started at its newest
+ * event, the cursor of that position.
+ */
+type DaemonEventPage =
+  | {
+      readonly kind: 'read';
+      readonly events: readonly Readonly<Record<string, unknown>>[];
+      readonly cursor: string;
+      readonly more: boolean;
+    }
+  | { readonly kind: 'unavailable' }
+  | { readonly kind: 'started'; readonly cursor: string };
+
+/**
+ * One daemon's part of a merge: the daemon, where its part of the cursor
+ * stood before the read (absent for a daemon the cursor left out, null for
+ * a read of its latest events), and what it gave. `unstarted` marks a page
+ * of the latest events of a daemon that had not answered since the read
+ * that started the cursor, and whether older events precede that page.
+ */
+export interface MergeSource {
+  readonly daemon: Pick<RegistryDaemon, 'name' | 'incarnation'>;
+  readonly before: { readonly cursor: string | null } | null;
+  readonly page: DaemonEventPage;
+  readonly unstarted?: { readonly olderUnread: boolean };
+}
+
+interface MergedEvents {
+  readonly events: readonly Readonly<Record<string, unknown>>[];
+  readonly cursor: string;
+  readonly more: boolean;
+  readonly unavailable: readonly string[];
+  readonly started: readonly string[];
+  readonly truncated: readonly string[];
+}
+
+// The id fields of one event, relative to the event.
+const EVENT_RULES: ReadonlyMap<string, IDRule> = new Map([
+  ['session', 'id'],
+  ['message', 'id'],
+  ['parent', 'id'],
+  ['report', 'id'],
+]);
+
+/**
+ * Merges the daemons' pages of one events read into one page of at most
+ * `limit` events. Each daemon's events keep their own order, and daemons
+ * interleave by timestamp as a best effort, since daemon clocks can skew.
+ * Each daemon's part of the returned cursor advances only past its events
+ * that made it into the page, which are always a prefix of what it gave,
+ * so an event read but cut is read again next time. A daemon that did not
+ * answer keeps its part and is listed under `unavailable`, and one that had
+ * no position yet keeps a null part, so the next read starts it at its
+ * latest events rather than skipping what it queued meanwhile. A daemon
+ * that started at its newest event, or at its latest events after such a
+ * gap, is listed under `started`, and under `truncated` too when older
+ * events precede the latest page and went unread. A report event also
+ * holds `report`, its daemon-qualified handle for `report.get`. Every event's
+ * ids are rewritten for its daemon, and its `cursor` is the gateway cursor
+ * that resumes right after it.
+ */
+export function mergeEventPages(
+  sources: readonly MergeSource[],
+  filter: string,
+  limit: number,
+): MergedEvents {
+  const parts = new Map<string, string | null>();
+
+  const queues: {
+    readonly source: MergeSource;
+    readonly events: readonly Readonly<Record<string, unknown>>[];
+    taken: number;
+  }[] = [];
+
+  const unavailable: string[] = [];
+  const started: string[] = [];
+  const truncated: string[] = [];
+
+  for (const source of sources) {
+    const key = `${source.daemon.name}.${source.daemon.incarnation}`;
+    const page = source.page;
+
+    if (page.kind === 'unavailable') {
+      unavailable.push(source.daemon.name);
+
+      if (source.before !== null) {
+        parts.set(key, source.before.cursor);
+      }
+    } else if (page.kind === 'started') {
+      started.push(source.daemon.name);
+      parts.set(key, page.cursor);
+    } else {
+      if (source.unstarted !== undefined) {
+        started.push(source.daemon.name);
+      }
+
+      if (source.unstarted?.olderUnread === true) {
+        truncated.push(source.daemon.name);
+      }
+
+      // An empty page's cursor is where the daemon stands. Until one of a
+      // page's events goes out, its daemon resumes right before the first
+      // one, a position the daemon reads as concrete, unlike no cursor,
+      // which it reads as its latest events.
+      const [first] = page.events;
+      const position = first === undefined ? page.cursor : buildPositionBefore(first);
+
+      parts.set(key, position);
+      queues.push({ source, events: page.events, taken: 0 });
+    }
+  }
+
+  const events: Readonly<Record<string, unknown>>[] = [];
+
+  while (events.length < limit) {
+    const next = pickNextQueue(queues);
+
+    if (next === null) {
+      break;
+    }
+
+    const event = next.events[next.taken] ?? {};
+    const daemon = next.source.daemon;
+
+    next.taken++;
+
+    if (typeof event['cursor'] === 'string') {
+      parts.set(`${daemon.name}.${daemon.incarnation}`, event['cursor']);
+    }
+
+    const rewritten = buildRuledValue(event, EVENT_RULES, daemon);
+    const handle = findReportHandle(event, daemon);
+
+    events.push({
+      ...(isRecord(rewritten) ? rewritten : event),
+      cursor: encodeGatewayCursor(filter, parts),
+      ...(handle === null ? {} : { report: handle }),
+    });
+  }
+
+  const more = queues.some(
+    (queue) =>
+      queue.taken < queue.events.length ||
+      (queue.source.page.kind === 'read' && queue.source.page.more),
+  );
+
+  return {
+    events,
+    cursor: encodeGatewayCursor(filter, parts),
+    more,
+    unavailable,
+    started,
+    truncated,
+  };
+}
+
+// A report event's handle for `report.get`: the daemon's own cursor of the
+// event, which the daemon reads a report by, under the daemon's name and
+// incarnation, kept apart from the merged feed cursor that replaces it.
+function findReportHandle(
+  event: Readonly<Record<string, unknown>>,
+  daemon: Pick<RegistryDaemon, 'name' | 'incarnation'>,
+): string | null {
+  const raw = event['cursor'];
+
+  return event['kind'] === 'report' && typeof raw === 'string' ? buildGatewayID(daemon, raw) : null;
+}
+
+// The queue whose next event goes out next: the one with the earliest
+// timestamp, ties going to the daemon that sorts first by name, or null
+// once every queue is spent.
+function pickNextQueue<
+  T extends {
+    readonly source: MergeSource;
+    readonly events: readonly Readonly<Record<string, unknown>>[];
+    readonly taken: number;
+  },
+>(queues: readonly T[]): T | null {
+  let best: T | null = null;
+  let bestAt = Number.POSITIVE_INFINITY;
+
+  for (const queue of queues) {
+    const head = queue.events[queue.taken];
+
+    if (head === undefined) {
+      continue;
+    }
+
+    const at = typeof head['at'] === 'number' ? head['at'] : Number.POSITIVE_INFINITY;
+
+    if (
+      best === null ||
+      at < bestAt ||
+      (at === bestAt && queue.source.daemon.name < best.source.daemon.name)
+    ) {
+      best = queue;
+      bestAt = at;
+    }
+  }
+
+  return best;
+}
+
+// The daemon cursor that resumes a read right before an event, from the
+// event's own cursor. Throws for an event whose cursor is not a daemon's
+// events cursor, which no daemon sends.
+function buildPositionBefore(event: Readonly<Record<string, unknown>>): string {
+  const raw = event['cursor'];
+  const decoded = typeof raw === 'string' ? decodeCursor(raw) : null;
+
+  if (decoded === null || decoded.kind !== 'events') {
+    throw new Error('a daemon event holds no events cursor');
+  }
+
+  return encodeCursor({ kind: 'events', id: decoded.id - 1 });
+}

package/src/federation/open-gateway-caller.ts

@@ -0,0 +1,55 @@
+import type { GatewayChannel } from './daemon-caller';
+import { DaemonPool } from './daemon-pool';
+import { GatewayStore } from './gateway-store';
+import { RoutingCaller } from './routing-caller';
+import type { GatewayRegistry, RegistryDaemon } from './types';
+
+interface GatewayCallerOptions {
+  readonly registry: GatewayRegistry;
+  readonly build: string;
+
+  // Connects to a daemon's TCP address.
+  readonly openChannel: (address: RegistryDaemon['address']) => Promise<GatewayChannel>;
+
+  // The keyed-request bindings.
+  readonly gatewayDBPath: string;
+
+  // How long each daemon may take to answer a call asked of every daemon.
+  readonly fanOutTimeoutMs?: number;
+}
+
+/**
+ * The caller the gateway's MCP tools ride: one pooled connection per
+ * registry daemon, the binding store, and the router over them, which the
+ * entrypoint hands to the MCP HTTP server. Nothing dials a daemon until a
+ * call needs it, so a daemon that is down never holds up the start. `stop`
+ * closes every daemon connection and the store.
+ */
+export function openGatewayCaller(opts: GatewayCallerOptions): {
+  readonly caller: RoutingCaller;
+  readonly stop: () => Promise<void>;
+} {
+  const store = GatewayStore.open(opts.gatewayDBPath);
+
+  const pool = new DaemonPool({
+    registry: opts.registry,
+    build: opts.build,
+    openChannel: opts.openChannel,
+  });
+
+  const caller = new RoutingCaller({
+    registry: opts.registry,
+    pool,
+    store,
+    ...(opts.fanOutTimeoutMs === undefined ? {} : { fanOutTimeoutMs: opts.fanOutTimeoutMs }),
+  });
+
+  return {
+    caller,
+    stop: async () => {
+      await pool.stop();
+
+      store.stop();
+    },
+  };
+}