@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/worker-host-checks.ts

@@ -4,14 +4,28 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
 import { verifyResourceSafety } from './platform/resource-verify';
 import policy from '../ansible/ubuntu/policy.json';
 import { userInfo } from 'node:os';
-import { T3_HEALTH_SCRIPT, runningVersionCheck } from './platform/t3-service';
+import {
+  T3_HEALTH_SCRIPT,
+  runningVersionCheck,
+  t3Unit,
+} from './platform/t3-service';
+import {
+  DEFAULT_FEATURES,
+  featuresCheck,
+  notApplicable,
+  offReason,
+  partsOff,
+  presentParts,
+  type WorkerFeatures,
+} from './worker-features';
+import { detectMachineClass, type MachineClass } from './machine/machine-class';
 import { compareDebianVersions } from './platform/debian-version';
 import { packagePins, ubuntuRelease } from './platform/ubuntu-release';
 import type { Check, Runner } from './types';
 
 function check(id: string, ok: boolean, summary: string): Check {
   return {
-    id: `worker:${id}`,
+    id,
     status: ok ? 'pass' : 'fail',
     summary,
     ...(!ok
@@ -35,7 +49,7 @@ export function pinnedPackage(
     installed === undefined ? -1 : compareDebianVersions(installed, pinned);
   if (order <= 0)
     return check(
-      `package:${name}`,
+      `worker:package:${name}`,
       order === 0,
       order === 0
         ? 'Installed at the version the policy pins.'
@@ -69,25 +83,34 @@ export function serviceState(name: string, show: string): string {
   return `${code} ${name}.service (${state}, unit file ${file || 'missing'}, restart ${field('Restart') || 'unknown'})`;
 }
 
-/** Measured runtime evidence, independent of installer receipts. */
+/** Measured runtime evidence, independent of installer receipts. What a
+ * worker's declared features leave out, and what its class of machine does
+ * not have, is reported as not applicable, with the reason. */
 export async function workerHostChecks(
   user: string,
-  run: Runner
+  run: Runner,
+  features: WorkerFeatures = DEFAULT_FEATURES,
+  powerRecoveryNote?: string
 ): Promise<Check[]> {
   const audit = await auditUbuntu(run);
+  const off = partsOff(features);
+  const { machine, virtualization } = await detectMachineClass(run);
   const checks = [
     check(
-      'ubuntu',
+      'worker:ubuntu',
       audit.supported,
       'Required Ubuntu release and architecture.'
     ),
     check(
-      'sudo',
+      'worker:sudo',
       audit.passwordlessSudo,
       'Designated worker user has noninteractive sudo.'
     ),
     ...Object.entries(packagePins(ubuntuRelease(audit.release))).map(
-      ([name, version]) => pinnedPackage(name, version, audit.packages[name])
+      ([name, version]) =>
+        off.packages.includes(name)
+          ? notApplicable(`worker:package:${name}`, offReason(features, name))
+          : pinnedPackage(name, version, audit.packages[name])
     ),
     pinnedPackage(
       'tailscale',
@@ -96,6 +119,12 @@ export async function workerHostChecks(
     ),
   ];
   for (const name of policy.alwaysOnServices) {
+    if (off.services.includes(name)) {
+      checks.push(
+        notApplicable(`worker:service:${name}`, offReason(features, name))
+      );
+      continue;
+    }
     const unit = await run([
       'systemctl',
       'show',
@@ -113,7 +142,7 @@ export async function workerHostChecks(
       /^Restart=always$/m.test(unit.stdout);
     checks.push(
       check(
-        `service:${name}`,
+        `worker:service:${name}`,
         ok,
         ok
           ? 'Required system service is active, enabled for boot and restarts unconditionally.'
@@ -122,33 +151,48 @@ export async function workerHostChecks(
     );
   }
   checks.push(await rebootPendingCheck(audit.rebootPending, run));
-  checks.push(...(await recoveryChecks(run)));
-  checks.push(...(await verifyUbuntuBoot(user, run)));
+  checks.push(featuresCheck(features, await presentParts(run, user, features)));
+  checks.push(
+    ...(await recoveryChecks(run, features, {
+      machine,
+      virtualization,
+      ...(powerRecoveryNote ? { note: powerRecoveryNote } : {}),
+    }))
+  );
+  checks.push(...(await verifyUbuntuBoot(user, run, machine)));
   checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
   return checks;
 }
 
-async function recoveryChecks(run: Runner): Promise<Check[]> {
+async function recoveryChecks(
+  run: Runner,
+  features: WorkerFeatures,
+  host: { machine: MachineClass; virtualization: string; note?: string }
+): Promise<Check[]> {
   const checks: Check[] = [];
   const t3 = await run([
     'systemctl',
     '--user',
     'show',
-    't3code-desktop.service',
+    t3Unit(features.session),
     '--property=ActiveState,UnitFileState,Restart',
   ]);
   checks.push(
     check(
-      't3-recovery',
+      'worker:t3-recovery',
       t3.ok &&
         /^ActiveState=active$/m.test(t3.stdout) &&
         /^UnitFileState=enabled$/m.test(t3.stdout) &&
         /^Restart=(always|on-failure)$/m.test(t3.stdout),
-      'The T3 desktop app runs as an enabled session service with automatic crash recovery.'
+      features.session === 'headless'
+        ? 'The T3 server runs as an enabled lingering user service with automatic crash recovery.'
+        : 'The T3 desktop app runs as an enabled session service with automatic crash recovery.'
     )
   );
   checks.push(await t3HealthCheck(run));
-  checks.push(await runningVersionCheck(run, userInfo().uid));
+  checks.push(
+    await runningVersionCheck(run, userInfo().uid, Date.now(), features.session)
+  );
   const renewal = await run([
     'systemctl',
     '--user',
@@ -157,13 +201,20 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
   ]);
   checks.push(
     check(
-      'credential-renewal',
+      'worker:credential-renewal',
       renewal.ok,
       'Worker credential renewal timer is active.'
     )
   );
   checks.push(await serviceRecoveryCheck(run));
-  checks.push(await powerRecoveryCheck(run));
+  checks.push(
+    host.machine === 'virtual'
+      ? notApplicable(
+          'worker:power-loss-recovery',
+          `a virtual machine (${host.virtualization}) has no firmware its guest can read, and its provider restarts it after a power loss${host.note ? ` (${host.note})` : ''}`
+        )
+      : await powerRecoveryCheck(run)
+  );
   return checks;
 }
 
@@ -194,7 +245,7 @@ export async function serviceRecoveryCheck(run: Runner): Promise<Check> {
         ? `WORKER_RECOVERY_LAST_RUN_FAILED (${/^[\w-]+$/.test(result) ? result : 'unknown'})`
         : 'Worker service recovery timer is active and its last run succeeded.';
   return check(
-    'service-recovery',
+    'worker:service-recovery',
     timer.ok && script.ok && unit.ok && result === 'success',
     summary
   );
@@ -224,7 +275,7 @@ async function t3HealthCheck(run: Runner): Promise<Check> {
     /* Raw responses are never reported. */
   }
   return check(
-    't3-health',
+    'worker:t3-health',
     probe.ok && identified,
     'T3 local API answers with its environment identity within 10 seconds.'
   );

package/src/worker-manifest.ts

@@ -21,6 +21,7 @@ export interface WorkerManifest {
 /** The policy a worker's manifest is measured against. */
 interface UbuntuPolicy {
   packages: Record<string, string>;
+  releases: { release: string; packages?: Record<string, string> }[];
   alwaysOnServices: string[];
 }
 
@@ -41,6 +42,8 @@ export function manifestProbe(services: string[]): string {
     "mise ls --current --json 2>/dev/null | tr -d '\\n'; echo",
     'echo @units',
     `for unit in ${units}; do printf '%s=%s/%s\\n' "$unit" "$(systemctl is-enabled "$unit" 2>/dev/null)" "$(systemctl is-active "$unit" 2>/dev/null)"; done`,
+    'echo @release',
+    '(. /etc/os-release 2>/dev/null && echo "$VERSION_ID")',
   ].join('; ');
 }
 
@@ -95,6 +98,14 @@ export function parseManifest(stdout: string): WorkerManifest {
   };
 }
 
+/** The release the probe reported. It is kept apart from the manifest, so
+ * the manifest and its digest are what they were before releases differed. */
+export function manifestRelease(stdout: string): string | null {
+  const lines = stdout.split('\n').map((line) => line.trim());
+  const at = lines.indexOf('@release');
+  return (at >= 0 && lines.slice(at + 1).find(Boolean)) || null;
+}
+
 /** The same manifest always has the same digest. */
 export function manifestDigest(manifest: WorkerManifest): string {
   return createHash('sha256').update(JSON.stringify(manifest)).digest('hex');
@@ -133,10 +144,17 @@ export function diffManifests(
  */
 export function declaredDifferences(
   manifest: WorkerManifest,
-  policy: UbuntuPolicy
+  policy: UbuntuPolicy,
+  release: string | null = null
 ): ManifestDifference[] {
   const differences: ManifestDifference[] = [];
-  for (const [name, version] of Object.entries(policy.packages)) {
+  // The release's own pins over the common ones, as tasks/release.yml
+  // combines them.
+  const pins = {
+    ...policy.packages,
+    ...policy.releases.find((entry) => entry.release === release)?.packages,
+  };
+  for (const [name, version] of Object.entries(pins)) {
     const actual = manifest.packages[name] ?? null;
     if (actual !== version)
       differences.push({
@@ -191,11 +209,16 @@ export async function measureManifest(
   node: FleetNode,
   run: Runner,
   policy: UbuntuPolicy
-): Promise<{ manifest: WorkerManifest } | { check: Check }> {
+): Promise<
+  { manifest: WorkerManifest; release: string | null } | { check: Check }
+> {
   const result = await run(
     [...sshArgs(node), manifestProbe(policy.alwaysOnServices)],
     { timeoutMs: 60_000 }
   );
   if (!result.ok) return { check: sshFailure(result) };
-  return { manifest: parseManifest(result.stdout) };
+  return {
+    manifest: parseManifest(result.stdout),
+    release: manifestRelease(result.stdout),
+  };
 }

package/src/worker-parking.ts

@@ -0,0 +1,70 @@
+import type { TailnetComputer } from './tailscale';
+import type { Check } from './types';
+import { duration } from './worker-activity';
+
+/** A parked worker is stopped or destroyed on purpose: it is listed with
+ * the time since it was parked, never contacted, never the canary, and it
+ * does not change the exit code. The fleet state says which ones are parked
+ * (src/state/workers.ts).
+ */
+export function parkedCheck(id: string, parkedAt: string, now: Date): Check {
+  const since = Math.max(0, (now.getTime() - Date.parse(parkedAt)) / 1000);
+  return {
+    id: 'worker:parked',
+    status: 'not-applicable',
+    summary: `${id} is parked since ${parkedAt} (${duration(since)}): it is not contacted. devfleet fleet-node-resume --id ${id} brings it back.`,
+  };
+}
+
+/** What fleet-sync converges once parked workers are set aside: the fleet
+ * without them (so none is contacted, expected online or made the canary)
+ * and a not-applicable check for each, which never changes the exit code.
+ */
+export function withoutParked<
+  D extends { workers: { id: string }[] },
+  N extends { id: string },
+>(
+  definition: D,
+  nodes: N[],
+  parked: Record<string, string>,
+  now: Date
+): { definition: D; nodes: N[]; checks: Check[] } {
+  return {
+    definition: {
+      ...definition,
+      workers: definition.workers.filter((worker) => !parked[worker.id]),
+    },
+    nodes: nodes.filter((node) => !parked[node.id]),
+    checks: definition.workers
+      .filter((worker) => parked[worker.id])
+      .map((worker) => parkedCheck(worker.id, parked[worker.id]!, now)),
+  };
+}
+
+/** A parked worker that is online on the tailnet anyway: it costs money
+ * while nobody uses it, or it was resumed by hand.
+ */
+export function parkedOnline(
+  id: string,
+  tailscaleName: string | undefined,
+  computers: TailnetComputer[]
+): Check | null {
+  if (!tailscaleName) return null;
+  const name = tailscaleName.toLowerCase();
+  const online = computers.some(
+    (computer) =>
+      !computer.self &&
+      computer.online &&
+      computer.dns.toLowerCase().replace(/\.$/, '') === name
+  );
+  return online
+    ? {
+        id: 'worker:parked-online',
+        status: 'warn',
+        summary: `${id} is parked, and online on the tailnet.`,
+        remedy: {
+          action: `Stop the machine if nobody uses it (it costs money while it runs), or resume it: devfleet fleet-node-resume --id ${id}.`,
+        },
+      }
+    : null;
+}

package/src/worker-policy.ts

@@ -2,6 +2,9 @@ import { CAPABILITIES, record } from './config';
 import { FleetUsageError } from './errors';
 import type { Capability } from './types';
 import { TOOL_PROFILES, type ToolProfile } from '../bootstrap/worker-tools';
+import { parseExposure, type ExposureDeclaration } from './exposure';
+import { featuresFault } from '../bootstrap/worker-features';
+import type { FeaturesDefinition } from './definition';
 
 export interface WorkerPolicy {
   requiredCapabilities: Capability[];
@@ -17,6 +20,10 @@ export interface WorkerPolicy {
   toolProfiles?: ToolProfile[];
   /** Single tools pinned in dev-fleet's mise.toml every worker installs. */
   tools?: string[];
+  /** What every worker opens beyond loopback and the tailnet (src/exposure.ts). */
+  exposure?: ExposureDeclaration;
+  /** Features every worker has unless its own `features` say otherwise. */
+  features?: FeaturesDefinition;
 }
 const REQUIRED = [
   'github',
@@ -47,6 +54,8 @@ const FIELDS: Record<string, (value: unknown) => boolean> = {
   neverSleep: (value) => value === true,
   toolProfiles: (value) => value === undefined || toolProfiles(value) !== null,
   tools: (value) => value === undefined || toolNames(value),
+  exposure: (value) => value === undefined || parseExposure(value) !== null,
+  features: (value) => value === undefined || featuresFault(value) === null,
 };
 /** The first field of a worker policy that breaks its rule, or null. */
 export function policyFault(value: Record<string, unknown>): string | null {
@@ -79,5 +88,12 @@ export function parseWorkerPolicy(value: unknown): WorkerPolicy {
     throw new FleetUsageError(
       'Invalid mandatory worker policy. Workers require the locked toolchain, common services, unattended sudo, persistent T3 and no sleep.'
     );
-  return value as unknown as WorkerPolicy;
+  const exposure =
+    value['exposure'] === undefined
+      ? undefined
+      : parseExposure(value['exposure']);
+  return {
+    ...(value as unknown as WorkerPolicy),
+    ...(exposure ? { exposure } : {}),
+  };
 }

package/src/worker-readiness.ts

@@ -6,16 +6,25 @@ import { workerSelection } from '../bootstrap/worker-tools';
 import { browserCheck, dockerRunCheck } from './worker-capability-checks';
 import { gitChecks } from './git-identity';
 import type { Check, Runner } from './types';
-import { workerHostChecks } from './worker-host-checks';
+import { UBUNTU_WORKER } from './platform/ubuntu-worker';
 import { workerAccessChecks } from './worker-access-checks';
 import { record } from './config';
+import { workerExposure } from './exposure';
+import { exposureCheck } from './worker-exposure';
+import { notApplicable, offReason, workerFeatures } from './worker-features';
 
 export async function workerReadiness(
   run: Runner,
   nodeId = hostId(hostname()),
   root = process.cwd()
 ): Promise<Check[]> {
-  const host = await workerHostChecks(userInfo().username, run);
+  const features = workerFeatures(fleet, nodeId);
+  const host = await UBUNTU_WORKER.hostChecks(
+    userInfo().username,
+    run,
+    features,
+    fleet.workers.find((entry) => entry.id === nodeId)?.powerRecoveryNote
+  );
   const access = await workerAccessChecks(run, nodeId);
   const revision = await run(['git', 'rev-parse', 'HEAD']);
   const singleton = await run(
@@ -62,11 +71,14 @@ export async function workerReadiness(
     ...host,
     ...access,
     proxy,
-    await dockerRunCheck(run),
+    features.containers
+      ? await dockerRunCheck(run)
+      : notApplicable('worker:docker-run', offReason(features, 'docker')),
     await browserCheck(
       run,
       workerSelection(fleet, nodeId).profiles.includes('web')
     ),
+    await exposureCheck(workerExposure(fleet, nodeId), run),
     ...(await gitChecks(fleet.git, run)),
     ...(await workerCapacity(root, nodeId, run)),
   ];

package/src/worker-reboot.ts

@@ -0,0 +1,129 @@
+import { playbookConnection } from './ssh';
+import type { PackagePlaybook } from './platform/ansible';
+import { blocking, type Check, type FleetNode } from './types';
+
+/** Why fleet-sync reboots a worker: its first converge, which nothing can be
+ * running on yet, or a maintenance reboot the operator asked for
+ * (`--maintenance-reboot`) to apply installed updates.
+ */
+export type RebootReason = 'first-converge' | 'maintenance';
+
+/** The boot-policy codes a reboot settles: the managed power files were
+ * written after this boot, or the running login manager has not loaded them.
+ */
+const SETTLED_BY_REBOOT = [
+  'UBUNTU_MAINTENANCE_REBOOT_PENDING',
+  'UBUNTU_SESSION_POLICY_PENDING',
+];
+
+/** The converge ran (its bootstrap passed) and says a reboot is needed. */
+export function rebootNeeded(checks: Check[]): boolean {
+  if (
+    checks.some((check) => check.id.startsWith('bootstrap:') && blocking(check))
+  )
+    return false;
+  return checks.some(
+    (check) =>
+      (check.id === 'ubuntu:boot-policy' &&
+        SETTLED_BY_REBOOT.includes(check.summary)) ||
+      (check.id === 'worker:reboot-pending' && check.status !== 'pass')
+  );
+}
+
+/** A maintenance run goes on to the next worker only once this one is back
+ * and ready: a reboot that failed, or one after which the worker is not
+ * ready, stops the rest.
+ */
+export function rebootStopsRollout(node: { checks: Check[] }): boolean {
+  const reboot = node.checks.find((check) => check.id === 'worker:reboot');
+  if (!reboot) return false;
+  if (reboot.summary === 'WORKER_REBOOT_FAILED') return true;
+  return reboot.status === 'pass' && node.checks.some(blocking);
+}
+
+/** Installed updates wait for a reboot: what --maintenance-reboot applies. */
+export function updatesPending(checks: Check[]): boolean {
+  return checks.some(
+    (check) => check.id === 'worker:reboot-pending' && check.status !== 'pass'
+  );
+}
+
+/** The home of the account a roster entry converges, from its workspace. */
+export function workerHome(node: FleetNode): string | null {
+  const suffix = '/.local/share/personal/fleet';
+  return node.workspace.endsWith(suffix)
+    ? node.workspace.slice(0, -suffix.length) || null
+    : null;
+}
+
+/** Reboots one worker through ansible/ubuntu/reboot.yml, run from the
+ * manager: only while its T3 runs no agent turn (unless `force`), waiting
+ * until it answers again. The play's output is read for its own codes only.
+ */
+export async function rebootWorker(
+  node: FleetNode,
+  request: { revision: string; reason: RebootReason; force?: boolean },
+  playbook: PackagePlaybook
+): Promise<Check> {
+  const home = workerHome(node);
+  if (!home)
+    return {
+      id: 'worker:reboot',
+      status: 'fail',
+      summary: 'WORKER_REBOOT_FAILED',
+      remedy: {
+        action:
+          'The roster names no workspace under the account home, so the worker was not rebooted. Rerun fleet-sync once it is enrolled.',
+      },
+    };
+  // `user@alias` only for a worker whose account this run created.
+  // A worker reached from declared data logs in as its declared account.
+  const connection = playbookConnection(node);
+  const [login, target] = node.ssh.includes('@')
+    ? node.ssh.split('@', 2)
+    : [connection.user ?? '', node.ssh];
+  const result = await playbook(
+    'ubuntu/reboot.yml',
+    {
+      target,
+      login,
+      home,
+      revision: request.revision,
+      reason: request.reason,
+      force: Boolean(request.force),
+      ssh_options: connection.options,
+    },
+    // The play waits up to 900 seconds for the worker to answer again.
+    1_200_000
+  );
+  if (result.ok)
+    return {
+      id: 'worker:reboot',
+      status: 'pass',
+      summary:
+        request.reason === 'first-converge'
+          ? 'Rebooted once after its first converge, which no agent turn ran on; it answered again and was converged once more.'
+          : 'Rebooted for maintenance while no agent turn ran; it answered again and was converged once more.',
+    };
+  if (result.stdout.includes('WORKER_REBOOT_BUSY'))
+    return {
+      id: 'worker:reboot',
+      // A busy worker is left alone, and the others still go: its boot
+      // policy or its pending updates still say a reboot is needed.
+      status: 'warn',
+      summary: 'WORKER_REBOOT_BUSY',
+      remedy: {
+        action:
+          'An agent turn is running on the worker, so it was not rebooted. Rerun fleet-sync --maintenance-reboot once the turn has ended, or add --force to end it.',
+      },
+    };
+  return {
+    id: 'worker:reboot',
+    status: 'fail',
+    summary: 'WORKER_REBOOT_FAILED',
+    remedy: {
+      action:
+        'The reboot did not complete, or the worker did not answer within 15 minutes. Check its console with the provider, then rerun fleet-sync.',
+    },
+  };
+}