@crvouga/dev-fleet 0.1.0 → 0.2.0

package/src/fleet-config.ts

@@ -7,8 +7,12 @@ import {
   writeFileSync,
 } from 'node:fs';
 import { homedir } from 'node:os';
-import { join } from 'node:path';
-import { parseDefinition, type FleetDefinition } from './definition';
+import { dirname, join } from 'node:path';
+import {
+  installedDefinition,
+  parseDefinition,
+  type FleetDefinition,
+} from './definition';
 import { FleetUsageError } from './errors';
 import { pinnedPath } from './agents/toolchain';
 import type { Runner } from './types';
@@ -17,14 +21,62 @@ import {
   type SecretCommand,
   type SecretsProvider,
 } from './secrets';
+import { isOperator, stateDefinition } from './state/operator';
+import type { BucketLocation } from './state/bucket';
+import { definitionPath, stateRoot } from './state/paths';
+
+/** Where every machine keeps the fleet configuration fleet-sync approved.
+ * Launchers and scheduled jobs get no environment and read it from here.
+ */
+const installedPath = (home: string): string =>
+  join(home, '.config/dev-fleet/fleet.json');
+
+/** Where this process reads the fleet configuration from, in this order:
+ * - `file`: the file `DEVFLEET_CONFIG` names. The devfleet CLI sets it to the
+ *   `fleet.json` of the project it is run from (src/cli/project.ts);
+ * - `state`: the `definition` document in the manager's fleet state, the
+ *   configuration fleet-sync last applied, which only the devfleet CLI
+ *   reads. `path` is its copy as a file;
+ * - `installed`: the copy fleet-sync installed on this machine.
+ */
+export interface DefinitionSource {
+  kind: 'file' | 'state' | 'installed';
+  path: string;
+}
 
-/** Where every machine keeps the fleet configuration. fleet-sync installs the
- * approved copy here, so launchers and scheduled jobs find it without any
- * environment. `DEVFLEET_CONFIG` names another file (the manager's checkout of
- * its configuration repository, or a test fixture).
+export function definitionSource(
+  env = process.env,
+  home = homedir()
+): DefinitionSource {
+  const root = stateRoot(env, home);
+  const copy = definitionPath(root);
+  const named = env['DEVFLEET_CONFIG'];
+  // A devfleet command names the state's copy for the commands it starts.
+  const inherited = named === copy && isOperator();
+  if (named && !inherited) return { kind: 'file', path: named };
+  if (stateDefinition(root) !== null) return { kind: 'state', path: copy };
+  return { kind: 'installed', path: installedPath(home) };
+}
+
+function writePrivate(path: string, content: string): boolean {
+  if (existsSync(path) && readFileSync(path, 'utf8') === content) return false;
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  writeFileSync(`${path}.new`, content, { mode: 0o600 });
+  renameSync(`${path}.new`, path);
+  return true;
+}
+
+/** The file that holds the configuration this process runs with, for the
+ * subprocesses that take a path. Every command this process starts reads the
+ * same one.
  */
 export function configPath(env = process.env, home = homedir()): string {
-  return env['DEVFLEET_CONFIG'] || join(home, '.config/dev-fleet/fleet.json');
+  const source = definitionSource(env, home);
+  if (source.kind !== 'state') return source.path;
+  const body = stateDefinition(stateRoot(env, home));
+  if (body !== null) writePrivate(source.path, body);
+  env['DEVFLEET_CONFIG'] = source.path;
+  return source.path;
 }
 
 /** Installs the configuration this process runs with as the machine's copy,
@@ -32,30 +84,64 @@ export function configPath(env = process.env, home = homedir()): string {
  * Returns whether the file changed.
  */
 export function installConfig(home = homedir()): boolean {
-  const target = join(home, '.config/dev-fleet/fleet.json');
-  const content = JSON.stringify(fleet, null, 2) + '\n';
-  if (existsSync(target) && readFileSync(target, 'utf8') === content)
-    return false;
-  mkdirSync(join(home, '.config/dev-fleet'), { recursive: true, mode: 0o700 });
-  writeFileSync(`${target}.new`, content, { mode: 0o600 });
-  renameSync(`${target}.new`, target);
-  return true;
+  return writePrivate(installedPath(home), installedDefinition(fleet));
+}
+
+function read(path: string): FleetDefinition {
+  let value: unknown;
+  try {
+    value = JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    throw new FleetUsageError(
+      `The fleet configuration at ${path} is not readable JSON.`
+    );
+  }
+  return parseDefinition(value);
 }
 
 function load(): FleetDefinition {
   const path = configPath();
   if (!existsSync(path))
     throw new FleetUsageError(
-      `No fleet configuration at ${path}. Copy examples/fleet.json, edit it for your machines, and set DEVFLEET_CONFIG or place it there.`
+      `No fleet configuration. Run devfleet from the project that holds your fleet.json (devfleet fleet-state --example prints one to start from), or name the file with DEVFLEET_CONFIG. A manager whose bucket already holds one restores it with devfleet fleet-state --init. Looked at ${path}.`
     );
-  return parseDefinition(JSON.parse(readFileSync(path, 'utf8')));
+  return read(path);
+}
+
+/** Where the configuration says the fleet state is kept, read from the file
+ * each time: null when it names no bucket, or when this machine has no
+ * configuration yet.
+ */
+export function declaredState(
+  env = process.env,
+  home = homedir()
+): BucketLocation | null {
+  const path = configPath(env, home);
+  return existsSync(path) ? (read(path).state ?? null) : null;
+}
+
+let loaded: FleetDefinition | undefined;
+const current = (): FleetDefinition => (loaded ??= load());
+
+/** Forgets the configuration read so far; the next use reads it again. For
+ * a command that has just replaced the fleet state under itself.
+ */
+export function reloadFleet(): void {
+  loaded = undefined;
 }
 
 /** This fleet's own configuration: its machines, endpoints, accounts and source
  * repository. Fleet code never names any of them; it reads them here, and this
- * is the only module that loads the file.
+ * is the only module that loads the file. It is read on first use, so a
+ * command that needs no configuration runs on a machine that has none yet.
  */
-export const fleet = load();
+export const fleet: FleetDefinition = new Proxy({} as FleetDefinition, {
+  get: (_, key) => Reflect.get(current(), key),
+  has: (_, key) => Reflect.has(current(), key),
+  ownKeys: () => Reflect.ownKeys(current()),
+  getOwnPropertyDescriptor: (_, key) =>
+    Reflect.getOwnPropertyDescriptor(current(), key),
+});
 
 /** Reads one of this fleet's secrets for this node. Tools run from the pinned
  * toolchain, whatever PATH the caller (T3, launchd, systemd) started with.

package/src/machine/source.ts

@@ -5,7 +5,7 @@ import type { Runner } from '../types';
 import { ensurePrivateRoot } from './state';
 import { fleet } from '../fleet-config';
 
-const REPOSITORY = fleet.source.repository;
+const repository = (): string => fleet.source.repository;
 
 /** Called after approval; local sources supply Git objects only, never dirty files. */
 export async function prepareSource(
@@ -40,7 +40,7 @@ async function prepareRemoteSource(
         '--no-checkout',
         '--origin',
         'origin',
-        REPOSITORY,
+        repository(),
         source,
       ],
       { timeoutMs: 120_000 }
@@ -48,7 +48,7 @@ async function prepareRemoteSource(
     if (!cloned.ok) return null;
     origin = await run(['git', 'remote', 'get-url', 'origin'], { cwd: source });
   }
-  if (!origin.ok || origin.stdout.trim() !== REPOSITORY) return null;
+  if (!origin.ok || origin.stdout.trim() !== repository()) return null;
   let found = await run(
     ['git', 'rev-parse', '--verify', `${revision}^{commit}`],
     { cwd: source }

package/src/machine/state.ts

@@ -25,14 +25,19 @@ export interface Enrollment {
 export const isRevision = (value: unknown): value is string =>
   typeof value === 'string' && /^[a-f0-9]{40}$/.test(value);
 
+/** The manager's enrollment from the file that held it before it moved into
+ * the fleet state (src/state/enrollment.ts). Nothing writes this file now.
+ */
 export async function readEnrollment(root: string): Promise<Enrollment | null> {
   try {
     const path = join(root, 'enrollment.json');
     const info = await lstat(path);
     if (!info.isFile() || info.size > 4096) throw new Error();
-    const data: unknown = JSON.parse(await readFile(path, 'utf8'));
-    if (!validEnrollment(data)) throw new Error();
-    return data as unknown as Enrollment;
+    const enrollment = parseEnrollment(
+      JSON.parse(await readFile(path, 'utf8'))
+    );
+    if (!enrollment) throw new Error();
+    return enrollment;
   } catch (error) {
     if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
     throw new FleetUsageError(
@@ -41,24 +46,6 @@ export async function readEnrollment(root: string): Promise<Enrollment | null> {
   }
 }
 
-export async function saveEnrollment(
-  root: string,
-  state: Enrollment
-): Promise<void> {
-  await mkdir(root, { recursive: true, mode: 0o700 });
-  const path = join(root, 'enrollment.json');
-  const temporary = `${path}.${randomUUID()}.tmp`;
-  try {
-    await writeFile(temporary, JSON.stringify(state) + '\n', {
-      mode: 0o600,
-      flag: 'wx',
-    });
-    await rename(temporary, path);
-  } finally {
-    await rm(temporary, { force: true });
-  }
-}
-
 /** A lock directory with no readable owner is only abandoned after this long,
  * so a run that has just created it but not yet written `owner.json` is safe. */
 const OWNERLESS_LOCK_GRACE_MS = 60_000;
@@ -169,17 +156,18 @@ export async function ensurePrivateRoot(root: string): Promise<void> {
   }
 }
 
-function validEnrollment(data: unknown): boolean {
-  if (!record(data) || data['version'] !== 1) return false;
+/** A valid enrollment record, or null. */
+export function parseEnrollment(data: unknown): Enrollment | null {
+  if (!record(data) || data['version'] !== 1) return null;
   if (
     typeof data['id'] !== 'string' ||
     !/^[a-z][a-z0-9-]{0,62}$/.test(data['id'])
   )
-    return false;
-  return (
-    isRevision(data['approvedRevision']) &&
+    return null;
+  return isRevision(data['approvedRevision']) &&
     ['appliedRevision', 'previousRevision'].every(
       (key) => data[key] === undefined || isRevision(data[key])
     )
-  );
+    ? (data as unknown as Enrollment)
+    : null;
 }

package/src/manager-sync.ts

@@ -10,7 +10,7 @@ import { discoverTailnet, type TailnetComputer } from './tailscale';
 import { matchComputers } from './tailnet-matching';
 import { sshArgs, sshFailure } from './ssh';
 import type { FleetDefinition, MachineDefinition } from './definition';
-import { PUBLIC_HUB_URL } from './monitoring/beszel';
+import { publicHubUrl } from './monitoring/beszel';
 import {
   convergeable,
   evaluatePlatform,
@@ -104,7 +104,7 @@ export async function synchronizeManager(
     desiredRevision: input.revision,
     checks: tailnet.checks,
     // Printed for people: the stable public name, never a tailnet address.
-    dashboardUrl: PUBLIC_HUB_URL,
+    dashboardUrl: publicHubUrl(),
     nodes: [],
     candidates: tailnet.computers.filter(
       (peer) => !peer.self && !matches.get(peer.id)?.length
@@ -179,7 +179,12 @@ async function applyNodes(
   deploy: DeployNode
 ): Promise<void> {
   const workers = report.nodes.filter((node) => node.role !== 'manager');
-  const canary = workers.find((node) => node.node === input.canary);
+  // A revision new to the fleet is proven on one worker first. When the run
+  // selects only later workers, the first selected one proves it instead:
+  // narrowing a run never skips the canary.
+  const canary = input.canary
+    ? (workers.find((node) => node.node === input.canary) ?? workers[0])
+    : undefined;
   let blocked = report.checks.some(blocking);
   const step = async (result: ManagerNode) => {
     const deployable = mayDeploy(result, input.nodes);

package/src/monitoring/agent-evidence.ts

@@ -0,0 +1,146 @@
+import type { Check, Runner } from '../types';
+
+/** Where installing the agent writes: the pinned build and its state. */
+export const AGENT_PATHS = ['/usr/local/lib', '/var/lib'] as const;
+/** Room for the archive, the extracted build and the agent's state. */
+export const AGENT_FREE_KIB = 256 * 1024;
+
+/** Measures, before anything is installed, that each path the agent writes
+ * to has room. A full disk is named as such instead of surfacing later as an
+ * incomplete installation.
+ */
+export async function agentDisk(run: Runner): Promise<Check> {
+  const result = await run(['df', '-Pk', ...AGENT_PATHS], {
+    timeoutMs: 10_000,
+  });
+  const free = result.ok ? available(result.stdout) : null;
+  if (!free || free.length !== AGENT_PATHS.length)
+    return {
+      id: 'beszel:disk',
+      status: 'fail',
+      summary: 'BESZEL_DISK_UNVERIFIED',
+      remedy: {
+        action: `Free space under ${AGENT_PATHS.join(' and ')} could not be measured. Check that df works for the fleet user, then rerun fleet-sync. Nothing was installed.`,
+      },
+    };
+  const least = Math.min(...free);
+  if (least < AGENT_FREE_KIB)
+    return {
+      id: 'beszel:disk',
+      status: 'fail',
+      summary: 'BESZEL_DISK_FULL',
+      remedy: {
+        action: `The monitoring agent needs ${AGENT_FREE_KIB / 1024} MiB free under each of ${AGENT_PATHS.join(' and ')}; the fullest has ${Math.floor(least / 1024)} MiB. Free space on the worker, then rerun fleet-sync. Nothing was installed and the running agent was left as it was.`,
+      },
+    };
+  return {
+    id: 'beszel:disk',
+    status: 'pass',
+    summary: `At least ${Math.floor(least / 1024)} MiB free where the agent is installed.`,
+  };
+}
+
+/** The "Available" column of POSIX df output, one value per path. */
+function available(output: string): number[] | null {
+  const rows = output.trim().split('\n').slice(1);
+  const values = rows.map((row) => Number(row.trim().split(/\s+/)[3]));
+  return values.length && values.every(Number.isSafeInteger) ? values : null;
+}
+
+/** What the pinned agent (Beszel 0.21.1) logs about its connection to the
+ * hub, from its source:
+ * - `agent/connection_manager.go` logs "WebSocket connected" once the hub
+ *   proved its key;
+ * - the hub answers an unknown token with 401 (`internal/hub/agent_connect.go`),
+ *   which the WebSocket library reports as "unexpected status code: 401";
+ * - a hub whose key is not the one the agent was given fails
+ *   `verifySignature`: "invalid signature - check KEY value";
+ * - a hub that is down behind its private route answers 502, 503 or 504.
+ * Only these fixed texts are matched; nothing of the log is reported.
+ */
+const OUTCOMES: { pattern: RegExp; check: Omit<Check, 'id'> }[] = [
+  {
+    pattern: /WebSocket connected/,
+    check: {
+      status: 'pass',
+      summary: 'The agent is connected to the hub, which proved its key.',
+    },
+  },
+  {
+    pattern: /unexpected status code: 401\b/,
+    check: {
+      status: 'fail',
+      summary: 'BESZEL_AGENT_TOKEN_REJECTED',
+      remedy: {
+        action:
+          "The hub refused this worker's agent token. Rerun fleet-sync from the manager so the worker's enrollment is resumed and its token delivered again. Never register a second record for the worker.",
+      },
+    },
+  },
+  {
+    pattern: /invalid signature - check KEY value/,
+    check: {
+      status: 'fail',
+      summary: 'BESZEL_AGENT_KEY_REJECTED',
+      remedy: {
+        action:
+          "The hub the agent reached did not prove the key it was given. Make sure the worker's agent points at the manager's hub, then rerun fleet-sync from the manager so the hub's key is delivered again.",
+      },
+    },
+  },
+  {
+    pattern: /unexpected status code: 50[234]\b/,
+    check: {
+      status: 'warn',
+      summary: 'BESZEL_HUB_UNAVAILABLE_TO_AGENT',
+      remedy: {
+        action:
+          'The private route answered but the hub behind it did not. Keep the manager awake with Docker Desktop running and rerun fleet-sync there; the agent reconnects by itself.',
+      },
+    },
+  },
+];
+
+/** The agent's newest connection outcome, read from its own log. The log
+ * holds host names, so only the outcome is reported.
+ */
+export async function agentConnection(run: Runner): Promise<Check> {
+  const result = await run(
+    [
+      'sudo',
+      '-n',
+      'journalctl',
+      '--unit=beszel-agent.service',
+      '--output=cat',
+      '--no-pager',
+      '--lines=200',
+    ],
+    { timeoutMs: 10_000 }
+  );
+  if (!result.ok)
+    return {
+      id: 'beszel:hub-connection',
+      status: 'skip',
+      summary: 'BESZEL_AGENT_LOG_UNAVAILABLE',
+      remedy: {
+        action:
+          "The agent's log could not be read, so how the hub answered it is unknown. The hub's own evidence in fleet-status --workers still applies.",
+      },
+    };
+  return { id: 'beszel:hub-connection', ...newestOutcome(result.stdout) };
+}
+
+function newestOutcome(log: string): Omit<Check, 'id'> {
+  const lines = log.split('\n').reverse();
+  for (const line of lines)
+    for (const outcome of OUTCOMES)
+      if (outcome.pattern.test(line)) return outcome.check;
+  return {
+    status: 'skip',
+    summary: 'BESZEL_AGENT_CONNECTION_PENDING',
+    remedy: {
+      action:
+        'The agent has not reached the hub yet. The next fleet-sync, or fleet-status --workers, reports whether it did.',
+    },
+  };
+}

package/src/monitoring/apply.ts

@@ -6,13 +6,15 @@ import { hubUrl, roleOf } from '../topology';
 import { record } from '../config';
 import { isRevision } from '../machine/state';
 import { privateHub } from './beszel';
+import { agentConnection, agentDisk } from './agent-evidence';
 import { runLockedCommand as runCommand } from '../runner';
 import type { Check, Runner } from '../types';
 
 export async function applyAgent(
   value: unknown,
   root: string,
-  run: Runner
+  run: Runner,
+  options: { receipts?: string } = {}
 ): Promise<Check[]> {
   const request = validateRequest(value);
   const definition = fleet;
@@ -32,7 +34,7 @@ export async function applyAgent(
   // Agents connect only to the manager's private hub URL.
   if (privateHub(request.hubUrl) !== privateHub(hubUrl(definition)))
     throw new Error('BESZEL_HUB_MISMATCH');
-  return installAgent(request, root, run);
+  return installAgent(request, root, run, options.receipts);
 }
 
 /** Workers (Linux) install a hardened systemd unit. The macOS manager's own
@@ -42,8 +44,12 @@ export async function applyAgent(
 async function installAgent(
   request: Request,
   root: string,
-  run: Runner
+  run: Runner,
+  receipts: string | undefined
 ): Promise<Check[]> {
+  // Measured before anything changes: a full disk is named, not guessed at.
+  const disk = await agentDisk(run);
+  if (disk.status !== 'pass') return [disk];
   const monitoring = resolve(root, 'ansible/monitoring');
   const result = await runAnsibleWithReceipt(
     run,
@@ -62,10 +68,12 @@ async function installAgent(
       name: 'beszel-agent',
       sources: [monitoring],
       verify: () => agentActive(run),
+      ...(receipts ? { directory: receipts } : {}),
     }
   );
   if (!result.ok)
     return [
+      disk,
       {
         id: 'beszel:installation',
         status: 'fail',
@@ -78,6 +86,7 @@ async function installAgent(
     ];
   const ready = await agentActive(run);
   return [
+    disk,
     {
       id: 'beszel:service',
       status: ready ? 'pass' : 'fail',
@@ -85,6 +94,9 @@ async function installAgent(
         ? 'Native Beszel service is enabled and active; hub metrics remain a separate gate.'
         : 'BESZEL_SERVICE_STOPPED',
     },
+    // How the hub answered the agent, from the agent's side: a rejected token
+    // or key is told apart from a hub that is merely down.
+    ...(ready ? [await agentConnection(run)] : []),
   ];
 }
 async function agentActive(run: Runner): Promise<boolean> {

package/src/monitoring/beszel.ts

@@ -1,4 +1,5 @@
 import { fleet } from '../fleet-config';
+import lock from '../../ansible/monitoring/lock.json';
 import type { Check } from '../types';
 
 export interface BeszelEvidence {
@@ -31,7 +32,7 @@ function check(
 /** Public dashboard hostname served through the Cloudflare
  * tunnel `beszel`. Agents and the manager keep using the private hub URL.
  */
-export const PUBLIC_HUB_URL = fleet.services.monitoring.publicUrl;
+export const publicHubUrl = (): string => fleet.services.monitoring.publicUrl;
 
 export function privateHub(value: string): string {
   const url = new URL(value);
@@ -49,7 +50,8 @@ export function privateHub(value: string): string {
   return url.origin;
 }
 
-/** Read-only PocketBase endpoints, verified against upstream Beszel v0.21.1.
+/** Read-only PocketBase endpoints, verified against the upstream Beszel
+ * release `ansible/monitoring/lock.json` pins.
  * Never return raw records: they may contain tokens, hosts or user information.
  */
 export async function verifyBeszel(
@@ -63,7 +65,7 @@ export async function verifyBeszel(
   const result: BeszelEvidence = { dashboardUrl, checks: [], receivedAt: null };
   try {
     const info = await request('/api/beszel/info');
-    if (!object(info) || info['v'] !== '0.21.1') {
+    if (!object(info) || info['v'] !== lock.version) {
       result.checks.push(
         check(
           'version',
@@ -75,7 +77,12 @@ export async function verifyBeszel(
       return result;
     }
     result.checks.push(
-      check('version', 'pass', 'Authenticated hub reports 0.21.1.', 'None.')
+      check(
+        'version',
+        'pass',
+        `Authenticated hub reports ${lock.version}.`,
+        'None.'
+      )
     );
     const filter = encodeURIComponent(`name="${node}"`);
     const records = items(

package/src/monitoring/enroll.ts

@@ -1,6 +1,7 @@
 import { createHash, randomUUID } from 'node:crypto';
 import { record } from '../config';
 import { privateHub } from './beszel';
+import lock from '../../ansible/monitoring/lock.json';
 
 export type HubRequest = (path: string, body?: object) => Promise<unknown>;
 export interface AgentEnrollment {
@@ -101,7 +102,7 @@ async function hubInfo(
   const info = await request('/api/beszel/info');
   if (
     !record(info) ||
-    info['v'] !== '0.21.1' ||
+    info['v'] !== lock.version ||
     typeof info['key'] !== 'string' ||
     !info['key'].startsWith('ssh-ed25519 ')
   )

package/src/monitoring/hub-access.ts

@@ -7,14 +7,15 @@ import { fleet, secret } from '../fleet-config';
  * Its password is the fleet's monitoring-account-password secret. A fresh hub on the
  * manager creates the account itself on first start (hub-darwin.yml).
  */
-export const BESZEL_FLEET_IDENTITY = fleet.services.monitoring.managerAccount;
+export const fleetIdentity = (): string =>
+  fleet.services.monitoring.managerAccount;
 
 /** Read-only Beszel user the hub signs every visitor in as (AUTO_LOGIN), so the
  * dashboard needs no login. Create it once in the hub (Settings > Users, role
  * "Read-only"); it holds no secret and must never be an admin. If the account
  * does not exist the hub simply shows its normal login page.
  */
-export const BESZEL_PUBLIC_VIEWER_IDENTITY =
+export const publicViewerIdentity = (): string =>
   fleet.services.monitoring.viewerAccount;
 
 /** Accept the plain password, or a legacy JSON identity/password pair. */
@@ -39,7 +40,7 @@ export function fleetCredentials(secret: string): {
       throw new Error('BESZEL_ACCOUNT_INVALID');
     return { identity: parsed['identity'], password: parsed['password'] };
   }
-  return { identity: BESZEL_FLEET_IDENTITY, password: value };
+  return { identity: fleetIdentity(), password: value };
 }
 
 /** The fleet login from the manager's Vault session, or null when absent.