@zgeoff/atc 3.1.1 → 3.3.0

package/src/verify-restored-fleet.ts

@@ -0,0 +1,155 @@
+import type { DaemonClient } from './client/daemon-client';
+import type { RestartFailedRow } from './parse-restart-result';
+import type { StoredRow } from './read-stored-rows';
+import { readStoredRows } from './read-stored-rows';
+import { isRecord } from './shared/report';
+
+interface FleetVerdict {
+  // The rows the daemon stored, exited ones included.
+  readonly total: number;
+  readonly failed: readonly RestartFailedRow[];
+}
+
+// The rows that are not restored yet, and whether waiting longer can still
+// change that.
+interface FailedRows {
+  readonly failed: readonly RestartFailedRow[];
+  readonly pending: boolean;
+}
+
+const UNSIZED_READ_MS = 30_000;
+
+/**
+ * Restores the stored fleet on a daemon and waits for every stored row to
+ * show up. `snapshot` holds the rows read from the old daemon before it
+ * stopped, with their original exit status; without one, the new daemon's
+ * rows are read instead. The call joins a restore the daemon already started by itself. A
+ * row that is not exited must be listed with a live terminal before the deadline
+ * passes (`timeoutSeconds` when set, else the rows that are not exited times the restore boot cap, plus 30 s), and an exited row must be listed. A stored row that is not listed
+ * failed to restore, and a listed row that is still without a live terminal
+ * at the deadline failed to revive. The deadline counts from the call, and
+ * every request to the daemon is held to it.
+ */
+export async function verifyRestoredFleet(
+  client: Pick<DaemonClient, 'sendRequest'>,
+  timeoutSeconds: number | null,
+  snapshot: readonly StoredRow[] | null,
+): Promise<FleetVerdict> {
+  const startedAt = Date.now();
+
+  // Without a snapshot the rows come from the new daemon, and its count is
+  // not known until they are read, so that read gets the fixed allowance.
+  const read =
+    snapshot === null
+      ? await sendBounded(
+          () => readStoredRows(client),
+          startedAt + (timeoutSeconds === null ? UNSIZED_READ_MS : timeoutSeconds * 1000),
+        )
+      : null;
+
+  const stored = snapshot ?? read ?? [];
+  const deadline = startedAt + pickDeadlineMs(stored, timeoutSeconds);
+
+  const restored = await tryRestore(client, deadline);
+  let found = await sendBounded(() => collectFailedRows(client, stored, restored), deadline);
+
+  while (found.pending && Date.now() < deadline) {
+    await Bun.sleep(250);
+
+    found = await sendBounded(() => collectFailedRows(client, stored, restored), deadline);
+  }
+
+  return { total: stored.length, failed: found.failed };
+}
+
+/**
+ * Runs a request and rejects once the deadline passes, so a daemon that
+ * stops answering cannot hold the restart, and its lock, open.
+ */
+async function sendBounded<T>(send: () => Promise<T>, deadline: number): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  const expired = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(
+      () => {
+        reject(new Error('the new daemon stopped answering before the restore deadline'));
+      },
+      Math.max(0, deadline - Date.now()),
+    );
+  });
+
+  try {
+    return await Promise.race([send(), expired]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function pickDeadlineMs(stored: readonly StoredRow[], timeoutSeconds: number | null): number {
+  if (timeoutSeconds !== null) {
+    return timeoutSeconds * 1000;
+  }
+
+  const cap = Number(process.env['ATC_RESTORE_BOOT_TIMEOUT_MS']);
+  const bootMs = Number.isFinite(cap) && cap >= 0 ? cap : 15_000;
+
+  return stored.filter((row) => !row.exited).length * bootMs + 30_000;
+}
+
+async function tryRestore(
+  client: Pick<DaemonClient, 'sendRequest'>,
+  deadline: number,
+): Promise<boolean> {
+  try {
+    await sendBounded(() => client.sendRequest('fleet.restore', { cols: 80, rows: 24 }), deadline);
+
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function collectFailedRows(
+  client: Pick<DaemonClient, 'sendRequest'>,
+  stored: readonly StoredRow[],
+  restored: boolean,
+): Promise<FailedRows> {
+  const listed = await client.sendRequest('session.list');
+
+  const sessions = Array.isArray(listed['sessions']) ? listed['sessions'].filter(isRecord) : [];
+
+  const byID = new Map(sessions.map((session) => [String(session['id']), session]));
+
+  const byAgentID = new Map(
+    sessions
+      .filter((session) => typeof session['agentSessionID'] === 'string')
+      .map((session) => [String(session['agentSessionID']), session]),
+  );
+
+  const failed: RestartFailedRow[] = [];
+  let pending = false;
+
+  for (const row of stored) {
+    const session =
+      byID.get(row.id) ??
+      (row.agentSessionID === null ? undefined : byAgentID.get(row.agentSessionID));
+
+    if (session === undefined) {
+      failed.push({ name: row.name, id: row.id, reason: 'not listed after the restore' });
+
+      // A finished restore lists every row it registers, so a row still
+      // missing never appears; after a failed restore it may yet.
+      pending ||= !restored;
+    } else if (!row.exited && !(session['alive'] === true && session['kind'] === 'pty')) {
+      failed.push({
+        name: row.name,
+        id: row.id,
+        reason: `listed in state ${String(session['state'])} without a terminal: ${String(session['lastMsg'])}`,
+      });
+
+      pending = true;
+    }
+  }
+
+  return { failed, pending };
+}

package/src/wait-for-replacement.ts

@@ -0,0 +1,112 @@
+import { DaemonClient } from './client/daemon-client';
+import { DaemonError } from './protocol/daemon-error';
+import { daemonRecordFile } from './shared/config';
+import { findDaemonRecord } from './shared/find-daemon-record';
+import { getBuild } from './shared/get-build';
+import { isProcessAlive } from './shared/is-process-alive';
+
+type ReplacementOutcome =
+  | {
+      readonly ok: true;
+      readonly client: DaemonClient;
+      readonly pid: number;
+      readonly build: string;
+      readonly listenPort: number | null;
+    }
+  | { readonly ok: false; readonly reason: string };
+
+/**
+ * Waits up to `waitMs` for a daemon to answer whose pid differs from the one
+ * that was stopped and whose record in the state directory matches it. With
+ * `expectedBuild`, the answering daemon must run that build: another client
+ * that won the start with a different build is a failure.
+ */
+export async function waitForReplacement(
+  oldPID: number | null,
+  expectedBuild: string | null,
+  waitMs: number,
+): Promise<ReplacementOutcome> {
+  const deadline = Date.now() + waitMs;
+  let refusal: string | null = null;
+
+  while (Date.now() < deadline) {
+    const record = findDaemonRecord(daemonRecordFile);
+
+    if (record !== null && record.pid !== oldPID && isProcessAlive(record.pid)) {
+      const attempt = await tryHello(record.socketPath, deadline);
+
+      if (attempt.kind === 'ok') {
+        if (expectedBuild !== null && attempt.build !== expectedBuild) {
+          attempt.client.stop();
+
+          return {
+            ok: false,
+            reason: `the daemon that came up (pid ${record.pid}) runs build ${attempt.build}, not ${expectedBuild}; another client started it first`,
+          };
+        }
+
+        return {
+          ok: true,
+          client: attempt.client,
+          pid: record.pid,
+          build: attempt.build,
+          listenPort: record.listenPort,
+        };
+      }
+
+      if (attempt.kind === 'refused') {
+        refusal = attempt.message;
+      }
+    }
+
+    await Bun.sleep(100);
+  }
+
+  return {
+    ok: false,
+    reason:
+      refusal === null
+        ? `no replacement daemon answered within ${Math.round(waitMs / 1000)} s`
+        : `the replacement daemon refused the handshake: ${refusal}`,
+  };
+}
+
+type Hello =
+  | { readonly kind: 'ok'; readonly client: DaemonClient; readonly build: string }
+  | { readonly kind: 'refused'; readonly message: string }
+  | { readonly kind: 'none' };
+
+async function tryHello(socketPath: string, deadline: number): Promise<Hello> {
+  let client: DaemonClient;
+
+  try {
+    client = await DaemonClient.open(socketPath);
+  } catch {
+    return { kind: 'none' };
+  }
+
+  // A daemon that accepts the socket and never answers must not outlast the
+  // startup deadline: closing the client rejects the handshake.
+  const expiry = setTimeout(
+    () => {
+      client.stop();
+    },
+    Math.max(0, deadline - Date.now()),
+  );
+
+  try {
+    const hello = await client.sendHello(getBuild());
+
+    const build = hello['daemon'];
+
+    return { kind: 'ok', client, build: typeof build === 'string' ? build : 'unknown' };
+  } catch (error) {
+    client.stop();
+
+    return error instanceof DaemonError && error.code === 'protocol_mismatch'
+      ? { kind: 'refused', message: error.message }
+      : { kind: 'none' };
+  } finally {
+    clearTimeout(expiry);
+  }
+}

package/src/write-restart-result.ts

@@ -0,0 +1,17 @@
+import { mkdirSync, renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { RestartResult } from './parse-restart-result';
+import { restartsDir } from './shared/config';
+
+/**
+ * Writes the result of a finished restart to `last.json` in the restarts
+ * directory, through a rename so a reader never sees half a record.
+ */
+export function writeRestartResult(result: RestartResult): void {
+  mkdirSync(restartsDir, { recursive: true });
+
+  const staging = join(restartsDir, `last.${process.pid}.tmp`);
+
+  writeFileSync(staging, `${JSON.stringify(result)}\n`);
+  renameSync(staging, join(restartsDir, 'last.json'));
+}