@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/public-reachability.ts

@@ -0,0 +1,186 @@
+import { createConnection, isIP } from 'node:net';
+import { record } from './config';
+import { fleet } from './fleet-config';
+import { workerExposure, type WorkerExposure } from './exposure';
+import { MONITORING_AGENT_PORT } from './worker-exposure';
+import type { Check, NodeReport, Runner } from './types';
+
+/** Opens one TCP connection and closes it at once: whether it was accepted. */
+export type Connect = (
+  host: string,
+  port: number,
+  timeoutMs: number
+) => Promise<boolean>;
+
+export const tcpConnect: Connect = (host, port, timeoutMs) =>
+  new Promise((resolve) => {
+    const socket = createConnection({ host, port });
+    const done = (accepted: boolean) => {
+      socket.destroy();
+      resolve(accepted);
+    };
+    socket.setTimeout(timeoutMs, () => done(false));
+    socket.once('connect', () => done(true));
+    socket.once('error', () => done(false));
+  });
+
+const UNRESTRICTED = 'WORKER_PUBLIC_PORT_UNRESTRICTED';
+
+const PRIVATE_V4: [number, number][] = [
+  [0x00000000, 8], // this network
+  [0x0a000000, 8],
+  [0x64400000, 10], // carrier-grade NAT and the tailnet
+  [0x7f000000, 8],
+  [0xa9fe0000, 16], // link-local
+  [0xac100000, 12],
+  [0xc0a80000, 16],
+  [0xe0000000, 3], // multicast and reserved
+];
+
+/** An address the internet routes to: not private, loopback, link-local,
+ * shared (which includes the tailnet), unique-local or multicast. */
+export function isPublicAddress(address: string): boolean {
+  if (isIP(address) === 4) {
+    const value =
+      address
+        .split('.')
+        .reduce((total, part) => total * 256 + Number(part), 0) >>> 0;
+    return !PRIVATE_V4.some(
+      ([network, bits]) =>
+        value >>> (32 - bits) === (network >>> (32 - bits)) >>> 0
+    );
+  }
+  if (isIP(address) === 6) {
+    const lower = address.toLowerCase();
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(lower);
+    if (mapped) return isPublicAddress(mapped[1]!);
+    return !(
+      lower === '::' ||
+      lower === '::1' ||
+      /^fe[89ab]/.test(lower) ||
+      /^f[cd]/.test(lower) ||
+      /^ff/.test(lower)
+    );
+  }
+  return false;
+}
+
+/** The public addresses Tailscale reaches a worker at (`Addrs` of its peer in
+ * `tailscale status --json`), each once. */
+export function publicEndpoints(
+  status: unknown,
+  tailscaleName: string
+): string[] {
+  if (!record(status) || !record(status['Peer'])) return [];
+  const name = tailscaleName.toLowerCase();
+  const peer = Object.values(status['Peer']).find(
+    (entry) =>
+      record(entry) &&
+      typeof entry['DNSName'] === 'string' &&
+      entry['DNSName'].replace(/\.$/, '').toLowerCase() === name
+  );
+  if (!record(peer) || !Array.isArray(peer['Addrs'])) return [];
+  const found = new Set<string>();
+  for (const entry of peer['Addrs'].slice(0, 32)) {
+    if (typeof entry !== 'string') continue;
+    const host = entry.replace(/:\d+$/, '').replace(/^\[|\]$/g, '');
+    if (isPublicAddress(host)) found.add(host);
+  }
+  return [...found];
+}
+
+/** worker:public-reachability, from the manager: none of the fleet's ports
+ * accepts a connection on the worker's public address. Optional: the manager
+ * may share the worker's network, so a failure here never blocks readiness;
+ * worker:exposure is measured on the worker itself. Names the service and
+ * port, never the address.
+ */
+export async function publicReachabilityCheck(
+  rules: WorkerExposure,
+  addresses: string[],
+  connect: Connect = tcpConnect
+): Promise<Check> {
+  if (addresses.length === 0)
+    return {
+      id: 'worker:public-reachability',
+      status: 'not-applicable',
+      required: false,
+      summary:
+        'Tailscale knows no public address for this worker: there is nothing outside the tailnet to try.',
+    };
+  const ports = [
+    ...(rules.policy.publicSsh ? [] : [{ name: 'SSH', port: rules.sshPort }]),
+    { name: 'T3', port: rules.t3Port },
+    { name: 'the monitoring agent', port: MONITORING_AGENT_PORT },
+  ].filter(
+    (entry) =>
+      !rules.policy.publicPorts.some(
+        (open) => open.protocol === 'tcp' && open.port === entry.port
+      )
+  );
+  const accepted: string[] = [];
+  for (const entry of ports)
+    for (const address of addresses)
+      if (await connect(address, entry.port, 3_000)) {
+        accepted.push(`${entry.name} (tcp ${entry.port})`);
+        break;
+      }
+  const tried = ports.map((entry) => `${entry.name} (tcp ${entry.port})`);
+  if (accepted.length)
+    return {
+      id: 'worker:public-reachability',
+      status: 'fail',
+      required: false,
+      summary: `${UNRESTRICTED} ${accepted.join(', ')} accepted a connection on the worker's public address.`,
+      remedy: {
+        action:
+          'Run devfleet fleet-sync, which loads the fleet firewall on the worker, then read worker:exposure for the unit that listens. When the manager shares the worker’s network, confirm from a machine outside it before changing anything.',
+      },
+    };
+  return {
+    id: 'worker:public-reachability',
+    status: 'pass',
+    required: false,
+    summary: `No connection was accepted on the worker's public address: ${tried.join(', ')}.`,
+  };
+}
+
+/** Adds worker:public-reachability to the report of every declared worker
+ * the tailnet knows. */
+export async function appendPublicReachability(
+  reports: NodeReport[],
+  run: Runner,
+  connect: Connect = tcpConnect
+): Promise<void> {
+  const status = await run(['tailscale', 'status', '--json'], {
+    timeoutMs: 15_000,
+  });
+  let parsed: unknown = null;
+  try {
+    parsed = status.ok ? JSON.parse(status.stdout) : null;
+  } catch {
+    /* Raw status is never reported. */
+  }
+  for (const machine of fleet.workers) {
+    const name = machine.network?.tailscaleName;
+    const report = reports.find(
+      (entry) => entry.node === name || entry.node === machine.id
+    );
+    if (!report || !name) continue;
+    report.checks.push(
+      parsed === null
+        ? {
+            id: 'worker:public-reachability',
+            status: 'skip',
+            required: false,
+            summary:
+              'Tailscale status is unavailable on the manager, so the public address of this worker is unknown.',
+          }
+        : await publicReachabilityCheck(
+            workerExposure(fleet, machine.id),
+            publicEndpoints(parsed, name),
+            connect
+          )
+    );
+  }
+}

package/src/remote-bootstrap.ts

@@ -24,6 +24,7 @@ import { record } from './config';
 import { installedDefinition, type FleetDefinition } from './definition';
 import { cleanCheck, validCheck } from './protocol';
 import { UBUNTU_RELEASES } from './platform/ubuntu-release';
+import { T3_RESTART_MODES } from './platform/t3-service';
 
 export interface BootstrapPayload {
   revision: string;
@@ -33,7 +34,12 @@ export interface BootstrapPayload {
   authorizeSudo?: string;
   /** The fleet configuration every node keeps at ~/.config/dev-fleet/fleet.json. */
   config?: string;
+  /** `fleet-sync --restart-busy`: a worker's T3 restarts for a new release or
+   * policy even while agent turns run. By default it waits for idle.
+   */
+  restartBusy?: boolean;
 }
+
 /** No manager tool installation; Git/source preparation is part of apply approval. */
 export async function prepareBootstrap(
   root: string,
@@ -107,16 +113,21 @@ async function bundleRevision(
     await unlink(path).catch(() => undefined);
   }
 }
+/** `tailnetKey` is a single-use enrolment key for a worker not on the tailnet
+ * yet. It travels on the session's stdin, after the configuration, and only
+ * to a bootstrap script that reads it: never as an argument.
+ */
 export async function bootstrapRemote(
   node: FleetNode,
   payload: BootstrapPayload,
   run: Runner,
-  follow: Follow = {}
+  options: Follow & { tailnetKey?: string } = {}
 ): Promise<NodeReport> {
+  const { tailnetKey, ...follow } = options;
   const clock = follow.clock ?? realClock;
-  const cached = await cachedRevision(node, payload.revision, run, clock);
-  if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
-  const transfer = cached
+  const held = await cachedRevision(node, payload.revision, run, clock);
+  if ('id' in held) return { node: node.id, checks: [held] };
+  const transfer = held.cached
     ? { mode: 'cached', bundle: '' }
     : { mode: 'transfer', bundle: payload.bundle };
   const command = [
@@ -129,14 +140,22 @@ export async function bootstrapRemote(
     node.id,
     transfer.mode,
     UBUNTU_RELEASES.map((entry) => entry.release).join(' '),
+    T3_RESTART_MODES[payload.restartBusy ? 1 : 0],
   ]
     .map(shellQuote)
     .join(' ');
   const checks = await bootstrapSessions(
     node,
     payload,
-    { command, input: bootstrapInput(payload.config, transfer.bundle) },
-    { run, follow: { ...follow, clock } }
+    {
+      command,
+      input: bootstrapInput(
+        payload.config,
+        transfer.bundle,
+        readsTailnetKey(payload.script) ? (tailnetKey ?? '') : null
+      ),
+    },
+    { run, follow: { ...follow, clock, earlier: held.earlier } }
   );
   return { node: node.id, checks };
 }
@@ -198,23 +217,37 @@ function locked(checks: Check[]): boolean {
 
 /** What the bootstrap reads on stdin: one line with the fleet configuration
  * (installed under the remote lock, so overlapping runs never both write it),
- * then the bundle when the worker lacks the revision.
+ * one line with the tailnet enrolment key (empty when there is none; absent
+ * for a script of an earlier revision, which reads no such line), then the
+ * bundle when the worker lacks the revision.
  */
 export function bootstrapInput(
   config: string | undefined,
-  bundle: string
+  bundle: string,
+  tailnetKey: string | null = null
 ): string {
   const line = config ? Buffer.from(config).toString('base64') : '';
-  return `${line}\n${bundle}`;
+  return tailnetKey === null
+    ? `${line}\n${bundle}`
+    : `${line}\n${tailnetKey}\n${bundle}`;
 }
 
+/** A bootstrap script reads the key line since the revision that added it. */
+export function readsTailnetKey(script: string): boolean {
+  return script.includes('read -r tailnet_key');
+}
+
+/** Whether the worker holds a clean checkout of the revision, and when the
+ * operation its record says had finished there started: that one ended
+ * before this run, so its result is never taken for this run's.
+ */
 async function cachedRevision(
   node: FleetNode,
   revision: string,
   run: Runner,
   clock: Clock
-): Promise<boolean | Check> {
-  const command = `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
+): Promise<{ cached: boolean; earlier: number | null } | Check> {
+  const command = `r="$HOME/.local/share/personal/fleet"; sed -n 's/^{.*"started":\\([0-9][0-9]*\\),.*"finished":[0-9].*$/finished=\\1/p' "$r/remote-operation.json" 2>/dev/null | head -n 1; d="$r/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
   const result = await retrying(
     () => run([...sshArgs(node), command], { timeoutMs: 20_000 }),
     clock
@@ -223,7 +256,12 @@ async function cachedRevision(
     const failure = sshFailure(result);
     if (result.reason !== 'exit' || failure.id !== 'ssh:probe') return failure;
   }
-  return result.ok && result.stdout === 'ready';
+  const lines = result.stdout.split('\n');
+  const finished = /^finished=(\d{1,12})$/.exec(lines[0] ?? '');
+  return {
+    cached: result.ok && lines.at(-1) === 'ready',
+    earlier: finished ? Number(finished[1]) : null,
+  };
 }
 
 function parseEvidence(result: {

package/src/remote-operation.ts

@@ -148,16 +148,30 @@ function followable(operation: Operation, revision: string): boolean {
   );
 }
 
-function finishedWith(operation: Operation, revision: string): Check[] | null {
+/** The result of a finished operation of this revision, unless it is the
+ * operation that had already finished before this run opened a session: a
+ * re-sync of the same revision finds the record of the last run still there,
+ * and its result is not this run's.
+ */
+function finishedWith(
+  operation: Operation,
+  revision: string,
+  earlier: number | null | undefined
+): Check[] | null {
   return !operation.held &&
     operation.finished &&
-    operation.revision === revision
+    operation.revision === revision &&
+    !(typeof earlier === 'number' && operation.started === earlier)
     ? operation.result
     : null;
 }
 
 export interface Follow {
   clock?: Clock;
+  /** When the operation that had finished on the worker before this run
+   * began had started there (the worker's clock, in seconds), when known.
+   */
+  earlier?: number | null;
   /** Told the phase and how long the operation has run, at every look. */
   progress?: (phase: string) => void;
 }
@@ -176,7 +190,7 @@ export async function followOperation(
   const deadline = clock.now() + FOLLOW_AT_MOST_MS;
   let operation = await inspectOperation(node, run, clock);
   if (!operation) return null;
-  const done = finishedWith(operation, revision);
+  const done = finishedWith(operation, revision, options.earlier);
   if (done) return [attached(operation, 'finished'), ...done];
   if (!followable(operation, revision)) return null;
   let last: Operation = operation;
@@ -186,7 +200,7 @@ export async function followOperation(
     operation = await inspectOperation(node, run, clock);
     if (!operation) continue;
     last = operation;
-    const result = finishedWith(operation, revision);
+    const result = finishedWith(operation, revision, options.earlier);
     if (result) return [attached(operation, 'followed'), ...result];
     if (!followable(operation, revision)) return null;
   }

package/src/repair-plan.ts

@@ -1,5 +1,5 @@
 import { checkOwner } from './check-owners';
-import { AUTOMATED_REPAIRS, recipe } from './repair-catalog';
+import { AUTOMATED_REPAIRS, REPAIR_RECIPES, recipe } from './repair-catalog';
 import type { RepairRecipe } from './repair-catalog';
 import type { Check, FleetNode, NodeReport, Fingerprint } from './types';
 
@@ -18,22 +18,33 @@ export interface RepairStep extends RepairRecipe {
   manual?: { check: string; reason: string }[];
 }
 
+const UNOWNED =
+  'No repair recipe is registered for this check: follow its remedy.';
+
+/** What repairs a check: the recipe its owner names, or why only a person
+ * can. An `auth:` check is repaired by the login recipe of its name; one
+ * with no such recipe is a person's, for the reason `login:*` gives. */
+export function checkRepair(
+  id: string
+): { recipe: string } | { manual: string } {
+  const owner = checkOwner(id);
+  if (!owner?.repair) return { manual: owner?.manual ?? UNOWNED };
+  if (owner.repair !== 'login:*') return { recipe: owner.repair };
+  const login = `login:${id.slice('auth:'.length)}`;
+  return REPAIR_RECIPES.some((entry) => entry.id === login)
+    ? { recipe: login }
+    : { manual: checkOwner(login)?.manual ?? UNOWNED };
+}
+
 export function repairId(check: Check): string {
-  const owner = checkOwner(check.id);
-  if (!owner?.repair) return 'operator';
-  return owner.repair === 'login:*'
-    ? `login:${check.id.slice('auth:'.length)}`
-    : owner.repair;
+  const repair = checkRepair(check.id);
+  return 'recipe' in repair ? repair.recipe : 'operator';
 }
 
 /** Why a check is left to a person, or null when a recipe repairs it. */
 export function manualReason(check: Check): string | null {
-  const owner = checkOwner(check.id);
-  if (owner?.repair) return null;
-  return (
-    owner?.manual ??
-    'No repair recipe is registered for this check: follow its remedy.'
-  );
+  const repair = checkRepair(check.id);
+  return 'manual' in repair ? repair.manual : null;
 }
 
 /** Every nonpassing check is accounted for; unknown checks remain operator work. */

package/src/report.ts

@@ -6,6 +6,7 @@ export function makeReport(command: string, nodes: NodeReport[]): Report {
     warn: 0,
     fail: 0,
     skip: 0,
+    'not-applicable': 0,
   };
   for (const node of nodes)
     for (const check of node.checks) summary[check.status] += 1;
@@ -46,7 +47,7 @@ export function renderReport(report: Report): string {
     }
   }
   lines.push(
-    `\n${report.summary.pass} pass, ${report.summary.warn} warn, ${report.summary.fail} fail, ${report.summary.skip} skip`
+    `\n${report.summary.pass} pass, ${report.summary.warn} warn, ${report.summary.fail} fail, ${report.summary.skip} skip${report.summary['not-applicable'] ? `, ${report.summary['not-applicable']} not applicable` : ''}`
   );
   return lines.join('\n');
 }

package/src/schedule-check.ts

@@ -1,8 +1,27 @@
+import macos from '../ansible/macos/policy.json';
+import type { ManagerDefinition } from './definition';
 import type { History } from './state/history';
 import type { Check } from './types';
 
-/** How often the manager's schedule runs fleet-sync. */
-export const SCHEDULE_INTERVAL_SECONDS = 1800;
+/** How often the manager's schedule runs fleet-sync by default. */
+export const SCHEDULE_INTERVAL_SECONDS = macos.fleetSync.intervalMinutes * 60;
+
+/** How often the manager's scheduled jobs run: `manager.schedule` in
+ * fleet.json, or the defaults of `ansible/macos/policy.json`.
+ */
+export function scheduleIntervals(
+  manager: Pick<ManagerDefinition, 'schedule'>
+): {
+  syncSeconds: number;
+  watchMinutes: number;
+} {
+  return {
+    syncSeconds:
+      (manager.schedule?.syncMinutes ?? macos.fleetSync.intervalMinutes) * 60,
+    watchMinutes:
+      manager.schedule?.watchMinutes ?? macos.fleetWatch.intervalMinutes,
+  };
+}
 
 /** `manager:schedule`: a schedule that is on must have succeeded within two
  * of its intervals. A schedule that was never turned on is no check at all.

package/src/secrets.ts

@@ -26,6 +26,13 @@ export const OPTIONAL_SECRET_NAMES = [
   // Where fleet-watch notifies (notify.targets).
   'notify-webhook-url',
   'notify-ntfy-url',
+  // A short-lived, single-use, tagged Tailscale auth key a new worker joins
+  // the tailnet with (fleet-node-seed).
+  'tailnet-auth-key',
+  // An API credential of the tailnet's control plane (tailnet in fleet.json)
+  // that creates one single-use key per new worker. It stays on the manager:
+  // no worker ever receives it.
+  'tailnet-admin',
 ] as const;
 export type SecretName =
   (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];

package/src/service-auth.ts

@@ -43,68 +43,75 @@ export async function codingAuth(
   ];
 }
 
-/** CLIs with no sign-in of their own to check, and why. Each says where
- * its access is proven instead; none is a silent skip.
+/** Registered agent CLIs with no sign-in of their own to check, and why.
+ * Each is `not-applicable` on the proxy route and says where its access is
+ * proven instead; none is a silent skip.
  */
 export const PROXY_ROUTED: Record<string, string> = {
   gemini:
     'gemini signs in through the CLI proxy and has no T3 launcher; provider:gemini:inference proves its route from this node.',
   opencode:
     'opencode signs in through the CLI proxy; agent:muse:session and the per-account agent:opencode sessions prove it from this node.',
+  grok: 'grok reads the proxy key from its environment and has no grok login; agent:grok:session proves it from this node.',
+  antigravity:
+    'antigravity signs in through the CLI proxy; agent:antigravity:session proves it from this node.',
 };
 
+/** The services a computer signs in to, each checked as `auth:<id>`. */
+const SERVICES: Service[] = [
+  {
+    id: 'railway',
+    command: ['railway', 'whoami', '--json'],
+    action: 'Run railway login on this computer.',
+    valid: (result) => typeof json(result)?.['id'] === 'string',
+  },
+  {
+    id: 'vercel',
+    command: ['vercel', 'whoami'],
+    action: 'Run vercel login on this computer.',
+    valid: (result) =>
+      /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,100}$/.test(result.stdout.trim()),
+  },
+  {
+    id: 'cloudflare',
+    command: ['wrangler', 'whoami', '--json'],
+    action: 'Run wrangler login on this computer.',
+    valid: (result) => json(result)?.['loggedIn'] === true,
+  },
+  {
+    id: 'aws',
+    command: [
+      'aws',
+      'sts',
+      'get-caller-identity',
+      '--output',
+      'json',
+      '--no-cli-pager',
+    ],
+    action:
+      'Select and authenticate the intended AWS profile on this computer.',
+    valid: (result) =>
+      typeof json(result)?.['Account'] === 'string' &&
+      typeof json(result)?.['Arn'] === 'string',
+  },
+  {
+    id: 'npm',
+    command: ['npm', 'whoami', '--registry=https://registry.npmjs.org/'],
+    action: 'Run npm login only if this computer should publish packages.',
+    valid: (result) =>
+      /^[a-z0-9][a-z0-9._-]{0,213}$/.test(result.stdout.trim()),
+  },
+];
+export const SERVICE_IDS = SERVICES.map((service) => service.id);
+
 export async function serviceAuth(run: Runner): Promise<Check[]> {
-  const services: Service[] = [
-    {
-      id: 'railway',
-      command: ['railway', 'whoami', '--json'],
-      action: 'Run railway login on this computer.',
-      valid: (result) => typeof json(result)?.['id'] === 'string',
-    },
-    {
-      id: 'vercel',
-      command: ['vercel', 'whoami'],
-      action: 'Run vercel login on this computer.',
-      valid: (result) =>
-        /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,100}$/.test(result.stdout.trim()),
-    },
-    {
-      id: 'cloudflare',
-      command: ['wrangler', 'whoami', '--json'],
-      action: 'Run wrangler login on this computer.',
-      valid: (result) => json(result)?.['loggedIn'] === true,
-    },
-    {
-      id: 'aws',
-      command: [
-        'aws',
-        'sts',
-        'get-caller-identity',
-        '--output',
-        'json',
-        '--no-cli-pager',
-      ],
-      action:
-        'Select and authenticate the intended AWS profile on this computer.',
-      valid: (result) =>
-        typeof json(result)?.['Account'] === 'string' &&
-        typeof json(result)?.['Arn'] === 'string',
-    },
-    {
-      id: 'npm',
-      command: ['npm', 'whoami', '--registry=https://registry.npmjs.org/'],
-      action: 'Run npm login only if this computer should publish packages.',
-      valid: (result) =>
-        /^[a-z0-9][a-z0-9._-]{0,213}$/.test(result.stdout.trim()),
-    },
-  ];
   const checks: Check[] = [];
-  for (const service of services) checks.push(await serviceCheck(service, run));
+  for (const service of SERVICES) checks.push(await serviceCheck(service, run));
   checks.push(await copilotCheck(run));
   for (const [id, reason] of Object.entries(PROXY_ROUTED))
     checks.push({
       id: `auth:${id}`,
-      status: 'skip',
+      status: 'not-applicable',
       required: false,
       summary: `AUTH_NOT_SUPPORTED_ON_ROUTE: ${reason}`,
     });