@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/remote-bootstrap.ts

@@ -4,7 +4,22 @@ import { join } from 'node:path';
 import { prepareSource } from './machine/source';
 import { isRevision } from './machine/state';
 import { sshArgs, shellQuote, sshFailure } from './ssh';
-import type { FleetNode, NodeReport, Runner, Check } from './types';
+import type {
+  FleetNode,
+  NodeReport,
+  Runner,
+  Check,
+  CommandResult,
+} from './types';
+import {
+  followOperation,
+  realClock,
+  retrying,
+  RETRY_PAUSES_MS,
+  transient,
+  type Clock,
+  type Follow,
+} from './remote-operation';
 import { record } from './config';
 import { installedDefinition, type FleetDefinition } from './definition';
 import { cleanCheck, validCheck } from './protocol';
@@ -94,9 +109,11 @@ async function bundleRevision(
 export async function bootstrapRemote(
   node: FleetNode,
   payload: BootstrapPayload,
-  run: Runner
+  run: Runner,
+  follow: Follow = {}
 ): Promise<NodeReport> {
-  const cached = await cachedRevision(node, payload.revision, run);
+  const clock = follow.clock ?? realClock;
+  const cached = await cachedRevision(node, payload.revision, run, clock);
   if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
   const transfer = cached
     ? { mode: 'cached', bundle: '' }
@@ -113,27 +130,68 @@ export async function bootstrapRemote(
   ]
     .map(shellQuote)
     .join(' ');
-  const result = await run([...sshArgs(node), command], {
-    input: bootstrapInput(payload.config, transfer.bundle),
-    timeoutMs: 5_400_000,
-  });
-  const evidence = parseEvidence(result);
-  if (evidence) return { node: node.id, checks: evidence };
-  if (!result.ok) return { node: node.id, checks: [sshFailure(result)] };
-  return {
-    node: node.id,
-    checks: [
-      {
-        id: 'bootstrap:protocol',
-        status: 'fail',
-        summary: 'REMOTE_BOOTSTRAP_EVIDENCE_INVALID',
-        remedy: {
-          action:
-            'Inspect the source-owned remote operation and lock. Do not assume a manager disconnect stopped installation.',
-        },
+  const checks = await bootstrapSessions(
+    node,
+    payload,
+    { command, input: bootstrapInput(payload.config, transfer.bundle) },
+    { run, follow: { ...follow, clock } }
+  );
+  return { node: node.id, checks };
+}
+
+/** Opens the bootstrap session, and again after a lost one unless the
+ * worker's own operation can be followed to its end instead.
+ */
+async function bootstrapSessions(
+  node: FleetNode,
+  payload: BootstrapPayload,
+  session: { command: string; input: string },
+  context: { run: Runner; follow: Follow & { clock: Clock } }
+): Promise<Check[]> {
+  const { run, follow } = context;
+  let result: CommandResult | undefined;
+  for (const pause of [0, ...RETRY_PAUSES_MS]) {
+    if (result) {
+      // The session was lost, not the operation: the worker keeps running
+      // it, so the manager follows it rather than starting another.
+      await follow.clock.pause(pause);
+      const followed = await followOperation(
+        node,
+        payload.revision,
+        run,
+        follow
+      );
+      if (followed) return followed;
+    }
+    result = await run([...sshArgs(node), session.command], {
+      input: session.input,
+      timeoutMs: 5_400_000,
+    });
+    const evidence = parseEvidence(result);
+    if (evidence && locked(evidence))
+      return (
+        (await followOperation(node, payload.revision, run, follow)) ?? evidence
+      );
+    if (evidence) return evidence;
+    if (result.reason !== 'exit' || !transient(result)) break;
+  }
+  if (!result!.ok) return [sshFailure(result!)];
+  return [
+    {
+      id: 'bootstrap:protocol',
+      status: 'fail',
+      summary: 'REMOTE_BOOTSTRAP_EVIDENCE_INVALID',
+      remedy: {
+        action:
+          'Inspect the source-owned remote operation and lock. Do not assume a manager disconnect stopped installation.',
       },
-    ],
-  };
+    },
+  ];
+}
+
+/** Another operation holds the worker: its lock was the first thing tried. */
+function locked(checks: Check[]): boolean {
+  return checks.some((check) => check.summary === 'FLEET_REMOTE_LOCKED');
 }
 
 /** What the bootstrap reads on stdin: one line with the fleet configuration
@@ -151,10 +209,14 @@ export function bootstrapInput(
 async function cachedRevision(
   node: FleetNode,
   revision: string,
-  run: Runner
+  run: Runner,
+  clock: Clock
 ): Promise<boolean | Check> {
   const command = `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
-  const result = await run([...sshArgs(node), command], { timeoutMs: 20_000 });
+  const result = await retrying(
+    () => run([...sshArgs(node), command], { timeoutMs: 20_000 }),
+    clock
+  );
   if (!result.ok) {
     const failure = sshFailure(result);
     if (result.reason !== 'exit' || failure.id !== 'ssh:probe') return failure;

package/src/remote-entry.ts

@@ -6,6 +6,8 @@ import { syncWorkspace } from './sync';
 import { repairWorkspace, validRepairs } from './repair-runner';
 import { validSnapshot } from './protocol';
 import type { Fingerprint } from './types';
+import { machineProviders } from './t3/providers';
+import { accountContext, nodeAccountChecks } from './accounts/context';
 
 async function main(): Promise<void> {
   const raw: unknown = JSON.parse(
@@ -36,6 +38,16 @@ async function main(): Promise<void> {
       raw['expected'] as Fingerprint
     );
     process.stdout.write(JSON.stringify({ version: 1, checks }));
+  } else if (raw['action'] === 'accounts') {
+    // The fast account path: only the account-derived T3 instances. T3
+    // watches its settings, so no restart and no running turn ends.
+    if (raw['apply'] !== true)
+      throw new Error('Account propagation requires explicit authorization.');
+    const checks = [
+      await machineProviders(node.id, node.workspace, true),
+      ...(await nodeAccountChecks(await accountContext())),
+    ];
+    process.stdout.write(JSON.stringify({ version: 1, checks }));
   } else if (raw['action'] === 'sync') {
     if (raw['apply'] !== true)
       throw new Error('Sync requires explicit authorization.');
@@ -84,6 +96,8 @@ async function configureRequest(
 function validRequest(raw: unknown): raw is Record<string, unknown> {
   return (
     record(raw) &&
-    ['probe', 'sync', 'repair', 'configure'].includes(String(raw['action']))
+    ['probe', 'sync', 'repair', 'configure', 'accounts'].includes(
+      String(raw['action'])
+    )
   );
 }

package/src/remote-operation.ts

@@ -0,0 +1,213 @@
+import { record } from './config';
+import { cleanCheck, validCheck } from './protocol';
+import { sshArgs, sshFailure } from './ssh';
+import type { Check, CommandResult, FleetNode, Runner } from './types';
+
+/** How the manager waits: a pause, and the time now. Tests answer both. */
+export interface Clock {
+  pause(ms: number): Promise<void>;
+  now(): number;
+}
+
+export const realClock: Clock = {
+  pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  now: () => Date.now(),
+};
+
+/** A session that could not be opened, or was lost, is tried again after
+ * these pauses: three attempts over thirty seconds.
+ */
+export const RETRY_PAUSES_MS = [10_000, 20_000];
+/** How often an operation the manager follows is looked at again. */
+export const FOLLOW_EVERY_MS = 15_000;
+/** As long as a bootstrap may run. */
+export const FOLLOW_AT_MOST_MS = 5_400_000;
+
+/** Failures of the connection, not of the worker: worth another attempt. A
+ * changed host key, a refused login or an unknown name never are.
+ */
+export function transient(result: CommandResult): boolean {
+  if (result.ok) return false;
+  const { id } = sshFailure(result);
+  return ['ssh:disconnected', 'ssh:network', 'ssh:timeout'].includes(id);
+}
+
+/** Runs a short SSH session, again after a pause while its connection fails. */
+export async function retrying(
+  attempt: () => Promise<CommandResult>,
+  clock: Clock
+): Promise<CommandResult> {
+  let result = await attempt();
+  for (const pause of RETRY_PAUSES_MS) {
+    if (!transient(result)) return result;
+    await clock.pause(pause);
+    result = await attempt();
+  }
+  return result;
+}
+
+/** What a worker says about the fleet operation on it. */
+export interface Operation {
+  /** The operation lock is held by a running process. */
+  held: boolean;
+  /** The process that recorded the operation is still running. */
+  alive: boolean;
+  revision: string | null;
+  phase: string | null;
+  /** Seconds since the epoch, on the worker's clock. */
+  started: number | null;
+  finished: boolean;
+  /** What node convergence reported, once the operation finished. */
+  result: Check[] | null;
+}
+
+/** Read-only. Prints whether the lock is held, whether the recorded process
+ * runs, the operation record and its result, one per line. Holding the lock
+ * for a moment to test it is not possible while another holds it, and takes
+ * nothing from anyone when nobody does.
+ */
+export const OPERATION_PROBE = [
+  'r="$HOME/.local/share/personal/fleet"',
+  'l="$r/remote-operation.lock"',
+  'held=no',
+  'if [ -f "$l" ] && [ ! -L "$l" ]; then if command -v flock >/dev/null 2>&1; then flock -n "$l" true || held=yes; else python3 -c \'import fcntl, sys; fcntl.flock(open(sys.argv[1], "a"), fcntl.LOCK_EX | fcntl.LOCK_NB)\' "$l" 2>/dev/null || held=yes; fi; fi',
+  'op=$(head -c 4096 "$r/remote-operation.json" 2>/dev/null | head -n 1)',
+  'pid=$(printf %s "$op" | sed -n \'s/.*"pid":\\([0-9][0-9]*\\).*/\\1/p\')',
+  'alive=no',
+  'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then alive=yes; fi',
+  'printf \'held=%s\\nalive=%s\\noperation=%s\\nresult=\' "$held" "$alive" "$op"',
+  'head -c 1048576 "$r/remote-operation-result.json" 2>/dev/null | head -n 1',
+  "printf '\\n'",
+].join('\n');
+
+function field(stdout: string, name: string): string | null {
+  const line = stdout.split('\n').find((entry) => entry.startsWith(`${name}=`));
+  return line === undefined ? null : line.slice(name.length + 1);
+}
+
+function json(text: string | null): unknown {
+  try {
+    return text ? JSON.parse(text) : null;
+  } catch {
+    return null;
+  }
+}
+
+function evidence(value: unknown): Check[] | null {
+  return record(value) &&
+    value['version'] === 1 &&
+    Array.isArray(value['checks']) &&
+    value['checks'].length > 0 &&
+    value['checks'].length <= 200 &&
+    value['checks'].every(validCheck)
+    ? value['checks'].map(cleanCheck)
+    : null;
+}
+
+export function parseOperation(stdout: string): Operation | null {
+  const held = field(stdout, 'held');
+  const alive = field(stdout, 'alive');
+  if (held === null || alive === null) return null;
+  const operation = json(field(stdout, 'operation'));
+  const op = record(operation) ? operation : {};
+  const text = (value: unknown, pattern: RegExp): string | null =>
+    typeof value === 'string' && pattern.test(value) ? value : null;
+  return {
+    held: held === 'yes',
+    alive: alive === 'yes',
+    revision: text(op['revision'], /^[a-f0-9]{40}$/),
+    phase: text(op['phase'], /^[a-z0-9-]{1,40}$/),
+    started: typeof op['started'] === 'number' ? op['started'] : null,
+    finished: typeof op['finished'] === 'number',
+    result: evidence(json(field(stdout, 'result'))),
+  };
+}
+
+export async function inspectOperation(
+  node: FleetNode,
+  run: Runner,
+  clock: Clock
+): Promise<Operation | null> {
+  const result = await retrying(
+    () => run([...sshArgs(node), OPERATION_PROBE], { timeoutMs: 20_000 }),
+    clock
+  );
+  return result.ok ? parseOperation(result.stdout) : null;
+}
+
+/** An operation of this revision whose process still runs: this is the run
+ * the manager lost, or one another manager run started; either way it is
+ * the run to follow, not one to start beside it.
+ */
+function followable(operation: Operation, revision: string): boolean {
+  return (
+    operation.held &&
+    operation.alive &&
+    !operation.finished &&
+    operation.revision === revision
+  );
+}
+
+function finishedWith(operation: Operation, revision: string): Check[] | null {
+  return !operation.held &&
+    operation.finished &&
+    operation.revision === revision
+    ? operation.result
+    : null;
+}
+
+export interface Follow {
+  clock?: Clock;
+  /** Told the phase and how long the operation has run, at every look. */
+  progress?: (phase: string) => void;
+}
+
+/** After a lost session or a held lock: the result of the operation of this
+ * revision on the worker, followed to its end when it still runs. Null when
+ * there is none to follow: another revision, another holder, or nothing.
+ */
+export async function followOperation(
+  node: FleetNode,
+  revision: string,
+  run: Runner,
+  options: Follow = {}
+): Promise<Check[] | null> {
+  const clock = options.clock ?? realClock;
+  const deadline = clock.now() + FOLLOW_AT_MOST_MS;
+  let operation = await inspectOperation(node, run, clock);
+  if (!operation) return null;
+  const done = finishedWith(operation, revision);
+  if (done) return [attached(operation, 'finished'), ...done];
+  if (!followable(operation, revision)) return null;
+  let last: Operation = operation;
+  while (clock.now() < deadline) {
+    options.progress?.(describe(last, clock));
+    await clock.pause(FOLLOW_EVERY_MS);
+    operation = await inspectOperation(node, run, clock);
+    if (!operation) continue;
+    last = operation;
+    const result = finishedWith(operation, revision);
+    if (result) return [attached(operation, 'followed'), ...result];
+    if (!followable(operation, revision)) return null;
+  }
+  return null;
+}
+
+function describe(operation: Operation, clock: Clock): string {
+  const elapsed =
+    operation.started === null
+      ? ''
+      : `, ${Math.max(0, Math.round(clock.now() / 1000 - operation.started))} s`;
+  return `Following the running operation (${operation.phase ?? 'unknown phase'}${elapsed})`;
+}
+
+function attached(_operation: Operation, how: 'finished' | 'followed'): Check {
+  return {
+    id: 'bootstrap:attached',
+    status: 'pass',
+    summary:
+      how === 'followed'
+        ? 'The manager lost or could not take this machine’s operation of the same revision, followed it to its end and reports its result.'
+        : 'This machine finished the operation of the same revision the manager lost; its result is reported.',
+  };
+}

package/src/schedule-check.ts

@@ -0,0 +1,40 @@
+import type { History } from './state/history';
+import type { Check } from './types';
+
+/** How often the manager's schedule runs fleet-sync. */
+export const SCHEDULE_INTERVAL_SECONDS = 1800;
+
+/** `manager:schedule`: a schedule that is on must have succeeded within two
+ * of its intervals. A schedule that was never turned on is no check at all.
+ */
+export function scheduleCheck(history: History, now: Date): Check | null {
+  const { schedule } = history;
+  if (!schedule?.enabled) return null;
+  const window = 2 * schedule.intervalSeconds * 1000;
+  const lastSuccess = history.runs
+    .filter((run) => run.trigger === 'schedule' && run.ok)
+    .map((run) => Date.parse(run.finishedAt))
+    .reduce((latest, at) => Math.max(latest, at), Number.NEGATIVE_INFINITY);
+  // A schedule turned on a moment ago has had no chance to run yet.
+  const since = Math.max(lastSuccess, Date.parse(schedule.changedAt));
+  const minutes = Math.round(schedule.intervalSeconds / 60);
+  if (now.getTime() - since <= window)
+    return {
+      id: 'manager:schedule',
+      status: 'pass',
+      summary: Number.isFinite(lastSuccess)
+        ? `The schedule last converged the fleet at ${new Date(lastSuccess).toISOString()}.`
+        : 'The schedule was turned on and has not had to run yet.',
+    };
+  return {
+    id: 'manager:schedule',
+    status: 'fail',
+    summary: Number.isFinite(lastSuccess)
+      ? `The schedule has not converged the fleet since ${new Date(lastSuccess).toISOString()}, more than two intervals of ${minutes} minutes.`
+      : `The schedule has not converged the fleet once since it was turned on, more than two intervals of ${minutes} minutes ago.`,
+    remedy: {
+      action:
+        'Run devfleet fleet-sync --schedule status on the manager to see the last scheduled runs, then devfleet fleet-report for the failing checks. Re-enable it with devfleet fleet-sync --schedule enable --apply --yes when the manager was replaced.',
+    },
+  };
+}

package/src/secrets.ts

@@ -12,7 +12,20 @@ export const SECRET_NAMES = [
   'monitoring-tunnel-token',
   'proxy-tunnel-token',
 ] as const;
-export type SecretName = (typeof SECRET_NAMES)[number];
+/** Secrets only a feature the fleet declares needs: a Vault location may
+ * leave them out, and they are then unavailable.
+ */
+export const OPTIONAL_SECRET_NAMES = [
+  // The off-machine storage of the monitoring hub's backups
+  // (services.monitoring.backups).
+  'monitoring-backup-access-key',
+  'monitoring-backup-secret-key',
+  // Where fleet-watch notifies (notify.targets).
+  'notify-webhook-url',
+  'notify-ntfy-url',
+] as const;
+export type SecretName =
+  (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];
 
 /** A secret's location in a Vault KV v2 mount. */
 export type VaultLocation = { path: string; field: string };
@@ -24,8 +37,13 @@ export type SecretsConfig =
       provider: 'vault';
       address: string;
       mount: string;
-      locations: Record<SecretName, VaultLocation>;
+      locations: VaultLocations;
     };
+export type VaultLocations = Record<
+  (typeof SECRET_NAMES)[number],
+  VaultLocation
+> &
+  Partial<Record<(typeof OPTIONAL_SECRET_NAMES)[number], VaultLocation>>;
 
 /** Runs a command and returns its stdout, or null when it fails. */
 export type SecretCommand = (
@@ -79,6 +97,7 @@ export function secretsProvider(
       );
   return async (name) => {
     const location = config.locations[name];
+    if (!location) return null;
     return present(
       await run([
         'vault',
@@ -142,13 +161,12 @@ const PARSERS: Record<string, (value: Fields) => SecretsConfig | null> = {
   },
 };
 
-function vaultLocations(
-  value: unknown
-): Record<SecretName, VaultLocation> | null {
+function vaultLocations(value: unknown): VaultLocations | null {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
   const fields = value as Fields;
-  if (!only(fields, [...SECRET_NAMES])) return null;
-  const valid = SECRET_NAMES.every((name) => {
+  if (!only(fields, [...SECRET_NAMES, ...OPTIONAL_SECRET_NAMES])) return null;
+  const optional = OPTIONAL_SECRET_NAMES.filter((name) => name in fields);
+  const valid = [...SECRET_NAMES, ...optional].every((name) => {
     const location = fields[name] as Fields | undefined;
     return (
       location !== null &&
@@ -158,7 +176,7 @@ function vaultLocations(
       text(location['field'])
     );
   });
-  return valid ? (fields as Record<SecretName, VaultLocation>) : null;
+  return valid ? (fields as VaultLocations) : null;
 }
 
 /** The `secrets` block of a fleet configuration, or null when it is invalid. */

package/src/service-auth.ts

@@ -42,6 +42,18 @@ export async function codingAuth(
   ];
 }
 
+/** CLIs with no sign-in of their own to check, and why. Each says where
+ * its access is proven instead; none is a silent skip.
+ */
+export const PROXY_ROUTED: Record<string, string> = {
+  gemini:
+    'gemini signs in through the CLI proxy and has no T3 launcher; provider:gemini:inference proves its route from this node.',
+  copilot:
+    'copilot is not routed through the CLI proxy and dev-fleet does not manage its GitHub sign-in; no provider account is checked.',
+  opencode:
+    'opencode signs in through the CLI proxy; agent:muse:session and the per-account agent:opencode sessions prove it from this node.',
+};
+
 export async function serviceAuth(run: Runner): Promise<Check[]> {
   const services: Service[] = [
     {
@@ -89,14 +101,12 @@ export async function serviceAuth(run: Runner): Promise<Check[]> {
   ];
   const checks: Check[] = [];
   for (const service of services) checks.push(await serviceCheck(service, run));
-  for (const id of ['gemini', 'copilot', 'opencode'])
+  for (const [id, reason] of Object.entries(PROXY_ROUTED))
     checks.push({
       id: `auth:${id}`,
       status: 'skip',
-      summary: `${id} provider authorization has no verified noninteractive status adapter in this CLI. Installed binaries or saved keys are not proof of access.`,
-      remedy: {
-        action: `Verify ${id} provider access locally using its supported account diagnostics. No prompt or billable request was made.`,
-      },
+      required: false,
+      summary: `AUTH_NOT_SUPPORTED_ON_ROUTE: ${reason}`,
     });
   return checks;
 }

package/src/ssh.ts

@@ -20,9 +20,9 @@ export function sshArgs(node: FleetNode): string[] {
     '-o',
     'ConnectionAttempts=1',
     '-o',
-    'ServerAliveInterval=5',
+    'ServerAliveInterval=15',
     '-o',
-    'ServerAliveCountMax=2',
+    'ServerAliveCountMax=4',
     '-o',
     'ClearAllForwardings=yes',
     '-o',
@@ -66,6 +66,14 @@ const FAILURES = [
     action:
       'Use ssh-add -l and ssh -vvv ALIAS locally; check User, IdentityFile, IdentitiesOnly, agent/keychain access and authorized_keys. Do not share debug logs without review.',
   },
+  {
+    pattern:
+      /Connection (?:to \S+ )?closed|Connection reset|Broken pipe|Timeout, server \S+ not responding|client_loop: send disconnect/i,
+    id: 'ssh:disconnected',
+    summary: 'The SSH session was lost while the operation ran.',
+    action:
+      'The worker keeps running what it started. Rerun fleet-sync: it follows the operation of the same revision to its end. Check the network between the manager and the worker if it repeats.',
+  },
   {
     pattern: /Connection refused|No route to host|Network is unreachable/i,
     id: 'ssh:network',