@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/worker-host-checks.ts

@@ -4,14 +4,28 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
 import { verifyResourceSafety } from './platform/resource-verify';
 import policy from '../ansible/ubuntu/policy.json';
 import { userInfo } from 'node:os';
-import { T3_HEALTH_SCRIPT, runningVersionCheck } from './platform/t3-service';
+import {
+  T3_HEALTH_SCRIPT,
+  runningVersionCheck,
+  t3Unit,
+} from './platform/t3-service';
+import {
+  DEFAULT_FEATURES,
+  featuresCheck,
+  notApplicable,
+  offReason,
+  partsOff,
+  presentParts,
+  type WorkerFeatures,
+} from './worker-features';
+import { detectMachineClass, type MachineClass } from './machine/machine-class';
 import { compareDebianVersions } from './platform/debian-version';
 import { packagePins, ubuntuRelease } from './platform/ubuntu-release';
 import type { Check, Runner } from './types';
 
 function check(id: string, ok: boolean, summary: string): Check {
   return {
-    id: `worker:${id}`,
+    id,
     status: ok ? 'pass' : 'fail',
     summary,
     ...(!ok
@@ -35,7 +49,7 @@ export function pinnedPackage(
     installed === undefined ? -1 : compareDebianVersions(installed, pinned);
   if (order <= 0)
     return check(
-      `package:${name}`,
+      `worker:package:${name}`,
       order === 0,
       order === 0
         ? 'Installed at the version the policy pins.'
@@ -69,25 +83,34 @@ export function serviceState(name: string, show: string): string {
   return `${code} ${name}.service (${state}, unit file ${file || 'missing'}, restart ${field('Restart') || 'unknown'})`;
 }
 
-/** Measured runtime evidence, independent of installer receipts. */
+/** Measured runtime evidence, independent of installer receipts. What a
+ * worker's declared features leave out, and what its class of machine does
+ * not have, is reported as not applicable, with the reason. */
 export async function workerHostChecks(
   user: string,
-  run: Runner
+  run: Runner,
+  features: WorkerFeatures = DEFAULT_FEATURES,
+  powerRecoveryNote?: string
 ): Promise<Check[]> {
   const audit = await auditUbuntu(run);
+  const off = partsOff(features);
+  const { machine, virtualization } = await detectMachineClass(run);
   const checks = [
     check(
-      'ubuntu',
+      'worker:ubuntu',
       audit.supported,
       'Required Ubuntu release and architecture.'
     ),
     check(
-      'sudo',
+      'worker:sudo',
       audit.passwordlessSudo,
       'Designated worker user has noninteractive sudo.'
     ),
     ...Object.entries(packagePins(ubuntuRelease(audit.release))).map(
-      ([name, version]) => pinnedPackage(name, version, audit.packages[name])
+      ([name, version]) =>
+        off.packages.includes(name)
+          ? notApplicable(`worker:package:${name}`, offReason(features, name))
+          : pinnedPackage(name, version, audit.packages[name])
     ),
     pinnedPackage(
       'tailscale',
@@ -96,6 +119,12 @@ export async function workerHostChecks(
     ),
   ];
   for (const name of policy.alwaysOnServices) {
+    if (off.services.includes(name)) {
+      checks.push(
+        notApplicable(`worker:service:${name}`, offReason(features, name))
+      );
+      continue;
+    }
     const unit = await run([
       'systemctl',
       'show',
@@ -113,7 +142,7 @@ export async function workerHostChecks(
       /^Restart=always$/m.test(unit.stdout);
     checks.push(
       check(
-        `service:${name}`,
+        `worker:service:${name}`,
         ok,
         ok
           ? 'Required system service is active, enabled for boot and restarts unconditionally.'
@@ -122,33 +151,48 @@ export async function workerHostChecks(
     );
   }
   checks.push(await rebootPendingCheck(audit.rebootPending, run));
-  checks.push(...(await recoveryChecks(run)));
-  checks.push(...(await verifyUbuntuBoot(user, run)));
+  checks.push(featuresCheck(features, await presentParts(run, user, features)));
+  checks.push(
+    ...(await recoveryChecks(run, features, {
+      machine,
+      virtualization,
+      ...(powerRecoveryNote ? { note: powerRecoveryNote } : {}),
+    }))
+  );
+  checks.push(...(await verifyUbuntuBoot(user, run, machine)));
   checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
   return checks;
 }
 
-async function recoveryChecks(run: Runner): Promise<Check[]> {
+async function recoveryChecks(
+  run: Runner,
+  features: WorkerFeatures,
+  host: { machine: MachineClass; virtualization: string; note?: string }
+): Promise<Check[]> {
   const checks: Check[] = [];
   const t3 = await run([
     'systemctl',
     '--user',
     'show',
-    't3code-desktop.service',
+    t3Unit(features.session),
     '--property=ActiveState,UnitFileState,Restart',
   ]);
   checks.push(
     check(
-      't3-recovery',
+      'worker:t3-recovery',
       t3.ok &&
         /^ActiveState=active$/m.test(t3.stdout) &&
         /^UnitFileState=enabled$/m.test(t3.stdout) &&
         /^Restart=(always|on-failure)$/m.test(t3.stdout),
-      'The T3 desktop app runs as an enabled session service with automatic crash recovery.'
+      features.session === 'headless'
+        ? 'The T3 server runs as an enabled lingering user service with automatic crash recovery.'
+        : 'The T3 desktop app runs as an enabled session service with automatic crash recovery.'
     )
   );
   checks.push(await t3HealthCheck(run));
-  checks.push(await runningVersionCheck(run, userInfo().uid));
+  checks.push(
+    await runningVersionCheck(run, userInfo().uid, Date.now(), features.session)
+  );
   const renewal = await run([
     'systemctl',
     '--user',
@@ -157,13 +201,20 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
   ]);
   checks.push(
     check(
-      'credential-renewal',
+      'worker:credential-renewal',
       renewal.ok,
       'Worker credential renewal timer is active.'
     )
   );
   checks.push(await serviceRecoveryCheck(run));
-  checks.push(await powerRecoveryCheck(run));
+  checks.push(
+    host.machine === 'virtual'
+      ? notApplicable(
+          'worker:power-loss-recovery',
+          `a virtual machine (${host.virtualization}) has no firmware its guest can read, and its provider restarts it after a power loss${host.note ? ` (${host.note})` : ''}`
+        )
+      : await powerRecoveryCheck(run)
+  );
   return checks;
 }
 
@@ -194,7 +245,7 @@ export async function serviceRecoveryCheck(run: Runner): Promise<Check> {
         ? `WORKER_RECOVERY_LAST_RUN_FAILED (${/^[\w-]+$/.test(result) ? result : 'unknown'})`
         : 'Worker service recovery timer is active and its last run succeeded.';
   return check(
-    'service-recovery',
+    'worker:service-recovery',
     timer.ok && script.ok && unit.ok && result === 'success',
     summary
   );
@@ -224,7 +275,7 @@ async function t3HealthCheck(run: Runner): Promise<Check> {
     /* Raw responses are never reported. */
   }
   return check(
-    't3-health',
+    'worker:t3-health',
     probe.ok && identified,
     'T3 local API answers with its environment identity within 10 seconds.'
   );

package/src/worker-manifest.ts

@@ -0,0 +1,224 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { PACKAGE_ROOT } from './package-info';
+import { shellQuote, sshArgs, sshFailure } from './ssh';
+import type { Check, FleetNode, Runner } from './types';
+
+/** What a worker is made of, measured: every installed package and its
+ * version, the tools mise has active, and whether each always-on service is
+ * enabled and running. Two workers built from the same revision have the same
+ * manifest, so its digest proves a rebuild reproduces a machine, and its
+ * difference from a fresh build shows what a long-lived worker gained.
+ */
+export interface WorkerManifest {
+  version: 1;
+  packages: Record<string, string>;
+  tools: Record<string, string>;
+  units: Record<string, string>;
+}
+
+/** The policy a worker's manifest is measured against. */
+interface UbuntuPolicy {
+  packages: Record<string, string>;
+  releases: { release: string; packages?: Record<string, string> }[];
+  alwaysOnServices: string[];
+}
+
+export async function ubuntuPolicy(root = PACKAGE_ROOT): Promise<UbuntuPolicy> {
+  return JSON.parse(
+    await readFile(join(root, 'ansible/ubuntu/policy.json'), 'utf8')
+  ) as UbuntuPolicy;
+}
+
+/** A read-only POSIX probe: it lists, and changes nothing. */
+export function manifestProbe(services: string[]): string {
+  const units = services.map(shellQuote).join(' ');
+  return [
+    'export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH"',
+    'echo @packages',
+    "dpkg-query -W -f='${binary:Package}=${Version}\\n' 2>/dev/null",
+    'echo @tools',
+    "mise ls --current --json 2>/dev/null | tr -d '\\n'; echo",
+    'echo @units',
+    `for unit in ${units}; do printf '%s=%s/%s\\n' "$unit" "$(systemctl is-enabled "$unit" 2>/dev/null)" "$(systemctl is-active "$unit" 2>/dev/null)"; done`,
+    'echo @release',
+    '(. /etc/os-release 2>/dev/null && echo "$VERSION_ID")',
+  ].join('; ');
+}
+
+const sorted = (entries: [string, string][]): Record<string, string> =>
+  Object.fromEntries(entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+
+function pairs(lines: string[]): [string, string][] {
+  return lines
+    .map((line) => line.trim())
+    .filter((line) => line.includes('='))
+    .map((line) => {
+      const at = line.indexOf('=');
+      return [line.slice(0, at), line.slice(at + 1)] as [string, string];
+    });
+}
+
+function tools(line: string | undefined): Record<string, string> {
+  try {
+    const listed = JSON.parse(line ?? '{}') as Record<
+      string,
+      { version?: string }[]
+    >;
+    return sorted(
+      Object.entries(listed).map(([tool, versions]) => [
+        tool,
+        versions
+          .map((entry) => entry.version ?? '')
+          .sort()
+          .join(','),
+      ])
+    );
+  } catch {
+    return {};
+  }
+}
+
+/** The manifest the probe printed. Order never matters: everything is sorted. */
+export function parseManifest(stdout: string): WorkerManifest {
+  const sections: Record<string, string[]> = {};
+  let current = '';
+  for (const line of stdout.split('\n')) {
+    if (/^@[a-z]+$/.test(line.trim())) {
+      current = line.trim().slice(1);
+      sections[current] = [];
+    } else if (current) sections[current]!.push(line);
+  }
+  return {
+    version: 1,
+    packages: sorted(pairs(sections['packages'] ?? [])),
+    tools: tools(sections['tools']?.find((line) => line.trim())),
+    units: sorted(pairs(sections['units'] ?? [])),
+  };
+}
+
+/** The release the probe reported. It is kept apart from the manifest, so
+ * the manifest and its digest are what they were before releases differed. */
+export function manifestRelease(stdout: string): string | null {
+  const lines = stdout.split('\n').map((line) => line.trim());
+  const at = lines.indexOf('@release');
+  return (at >= 0 && lines.slice(at + 1).find(Boolean)) || null;
+}
+
+/** The same manifest always has the same digest. */
+export function manifestDigest(manifest: WorkerManifest): string {
+  return createHash('sha256').update(JSON.stringify(manifest)).digest('hex');
+}
+
+export interface ManifestDifference {
+  section: 'packages' | 'tools' | 'units';
+  name: string;
+  expected: string | null;
+  actual: string | null;
+}
+
+/** What `live` has that `reference` does not, lacks, or has otherwise. */
+export function diffManifests(
+  reference: WorkerManifest,
+  live: WorkerManifest
+): ManifestDifference[] {
+  const differences: ManifestDifference[] = [];
+  for (const section of ['packages', 'tools', 'units'] as const) {
+    const names = new Set([
+      ...Object.keys(reference[section]),
+      ...Object.keys(live[section]),
+    ]);
+    for (const name of [...names].sort()) {
+      const expected = reference[section][name] ?? null;
+      const actual = live[section][name] ?? null;
+      if (expected !== actual)
+        differences.push({ section, name, expected, actual });
+    }
+  }
+  return differences;
+}
+
+/** Where a manifest departs from what the policy declares: a pinned package
+ * at another version or missing, an always-on service not enabled and active.
+ */
+export function declaredDifferences(
+  manifest: WorkerManifest,
+  policy: UbuntuPolicy,
+  release: string | null = null
+): ManifestDifference[] {
+  const differences: ManifestDifference[] = [];
+  // The release's own pins over the common ones, as tasks/release.yml
+  // combines them.
+  const pins = {
+    ...policy.packages,
+    ...policy.releases.find((entry) => entry.release === release)?.packages,
+  };
+  for (const [name, version] of Object.entries(pins)) {
+    const actual = manifest.packages[name] ?? null;
+    if (actual !== version)
+      differences.push({
+        section: 'packages',
+        name,
+        expected: version,
+        actual,
+      });
+  }
+  for (const unit of policy.alwaysOnServices) {
+    const actual = manifest.units[unit] ?? null;
+    if (actual !== 'enabled/active')
+      differences.push({
+        section: 'units',
+        name: unit,
+        expected: 'enabled/active',
+        actual,
+      });
+  }
+  return differences;
+}
+
+/** `worker:manifest`: the worker is what a fresh build of the same revision
+ * is (when a reference is given) and what the policy declares.
+ */
+export function manifestCheck(differences: ManifestDifference[]): Check {
+  if (!differences.length)
+    return {
+      id: 'worker:manifest',
+      status: 'pass',
+      summary: 'The worker matches the declared pins and the fresh build.',
+    };
+  const shown = differences
+    .slice(0, 20)
+    .map(
+      (difference) =>
+        `${difference.section}:${difference.name} ${difference.expected ?? 'absent'} -> ${difference.actual ?? 'absent'}`
+    );
+  return {
+    id: 'worker:manifest',
+    status: 'fail',
+    summary: `WORKER_MANIFEST_DRIFT: ${differences.length} difference(s): ${shown.join('; ')}`,
+    remedy: {
+      action:
+        'Something was installed, removed or changed outside fleet-sync. Rebuild the worker (fleet-node-replace) or declare the change in dev-fleet.',
+    },
+  };
+}
+
+/** Measures one worker's manifest over the verified SSH connection. */
+export async function measureManifest(
+  node: FleetNode,
+  run: Runner,
+  policy: UbuntuPolicy
+): Promise<
+  { manifest: WorkerManifest; release: string | null } | { check: Check }
+> {
+  const result = await run(
+    [...sshArgs(node), manifestProbe(policy.alwaysOnServices)],
+    { timeoutMs: 60_000 }
+  );
+  if (!result.ok) return { check: sshFailure(result) };
+  return {
+    manifest: parseManifest(result.stdout),
+    release: manifestRelease(result.stdout),
+  };
+}

package/src/worker-parking.ts

@@ -0,0 +1,70 @@
+import type { TailnetComputer } from './tailscale';
+import type { Check } from './types';
+import { duration } from './worker-activity';
+
+/** A parked worker is stopped or destroyed on purpose: it is listed with
+ * the time since it was parked, never contacted, never the canary, and it
+ * does not change the exit code. The fleet state says which ones are parked
+ * (src/state/workers.ts).
+ */
+export function parkedCheck(id: string, parkedAt: string, now: Date): Check {
+  const since = Math.max(0, (now.getTime() - Date.parse(parkedAt)) / 1000);
+  return {
+    id: 'worker:parked',
+    status: 'not-applicable',
+    summary: `${id} is parked since ${parkedAt} (${duration(since)}): it is not contacted. devfleet fleet-node-resume --id ${id} brings it back.`,
+  };
+}
+
+/** What fleet-sync converges once parked workers are set aside: the fleet
+ * without them (so none is contacted, expected online or made the canary)
+ * and a not-applicable check for each, which never changes the exit code.
+ */
+export function withoutParked<
+  D extends { workers: { id: string }[] },
+  N extends { id: string },
+>(
+  definition: D,
+  nodes: N[],
+  parked: Record<string, string>,
+  now: Date
+): { definition: D; nodes: N[]; checks: Check[] } {
+  return {
+    definition: {
+      ...definition,
+      workers: definition.workers.filter((worker) => !parked[worker.id]),
+    },
+    nodes: nodes.filter((node) => !parked[node.id]),
+    checks: definition.workers
+      .filter((worker) => parked[worker.id])
+      .map((worker) => parkedCheck(worker.id, parked[worker.id]!, now)),
+  };
+}
+
+/** A parked worker that is online on the tailnet anyway: it costs money
+ * while nobody uses it, or it was resumed by hand.
+ */
+export function parkedOnline(
+  id: string,
+  tailscaleName: string | undefined,
+  computers: TailnetComputer[]
+): Check | null {
+  if (!tailscaleName) return null;
+  const name = tailscaleName.toLowerCase();
+  const online = computers.some(
+    (computer) =>
+      !computer.self &&
+      computer.online &&
+      computer.dns.toLowerCase().replace(/\.$/, '') === name
+  );
+  return online
+    ? {
+        id: 'worker:parked-online',
+        status: 'warn',
+        summary: `${id} is parked, and online on the tailnet.`,
+        remedy: {
+          action: `Stop the machine if nobody uses it (it costs money while it runs), or resume it: devfleet fleet-node-resume --id ${id}.`,
+        },
+      }
+    : null;
+}

package/src/worker-policy.ts

@@ -2,6 +2,9 @@ import { CAPABILITIES, record } from './config';
 import { FleetUsageError } from './errors';
 import type { Capability } from './types';
 import { TOOL_PROFILES, type ToolProfile } from '../bootstrap/worker-tools';
+import { parseExposure, type ExposureDeclaration } from './exposure';
+import { featuresFault } from '../bootstrap/worker-features';
+import type { FeaturesDefinition } from './definition';
 
 export interface WorkerPolicy {
   requiredCapabilities: Capability[];
@@ -17,6 +20,10 @@ export interface WorkerPolicy {
   toolProfiles?: ToolProfile[];
   /** Single tools pinned in dev-fleet's mise.toml every worker installs. */
   tools?: string[];
+  /** What every worker opens beyond loopback and the tailnet (src/exposure.ts). */
+  exposure?: ExposureDeclaration;
+  /** Features every worker has unless its own `features` say otherwise. */
+  features?: FeaturesDefinition;
 }
 const REQUIRED = [
   'github',
@@ -34,33 +41,29 @@ function validCapabilities(value: unknown): boolean {
     REQUIRED.every((item) => value.includes(item))
   );
 }
-function validPolicy(value: Record<string, unknown>): boolean {
+const FIELDS: Record<string, (value: unknown) => boolean> = {
+  requiredCapabilities: validCapabilities,
+  githubLogin: (value) =>
+    typeof value === 'string' && /^[a-zA-Z0-9-]{1,39}$/.test(value),
+  vaultUser: (value) =>
+    typeof value === 'string' && /^[a-zA-Z0-9_.-]{1,64}$/.test(value),
+  toolchain: (value) => value === 'mise.lock',
+  agentProfile: (value) => value === 'proxy',
+  unattendedSudo: (value) => value === true,
+  persistentT3: (value) => value === true,
+  neverSleep: (value) => value === true,
+  toolProfiles: (value) => value === undefined || toolProfiles(value) !== null,
+  tools: (value) => value === undefined || toolNames(value),
+  exposure: (value) => value === undefined || parseExposure(value) !== null,
+  features: (value) => value === undefined || featuresFault(value) === null,
+};
+/** The first field of a worker policy that breaks its rule, or null. */
+export function policyFault(value: Record<string, unknown>): string | null {
+  const unknown = Object.keys(value).find((key) => !(key in FIELDS));
+  if (unknown !== undefined) return unknown;
   return (
-    Object.keys(value).every((key) =>
-      [
-        'requiredCapabilities',
-        'githubLogin',
-        'vaultUser',
-        'toolchain',
-        'agentProfile',
-        'unattendedSudo',
-        'persistentT3',
-        'neverSleep',
-        'toolProfiles',
-        'tools',
-      ].includes(key)
-    ) &&
-    validTools(value) &&
-    validCapabilities(value['requiredCapabilities']) &&
-    typeof value['githubLogin'] === 'string' &&
-    /^[a-zA-Z0-9-]{1,39}$/.test(value['githubLogin']) &&
-    typeof value['vaultUser'] === 'string' &&
-    /^[a-zA-Z0-9_.-]{1,64}$/.test(value['vaultUser']) &&
-    value['toolchain'] === 'mise.lock' &&
-    value['agentProfile'] === 'proxy' &&
-    ['unattendedSudo', 'persistentT3', 'neverSleep'].every(
-      (key) => value[key] === true
-    )
+    Object.entries(FIELDS).find(([key, valid]) => !valid(value[key]))?.[0] ??
+    null
   );
 }
 /** A list of known tool profiles, each once; null when it is not one. */
@@ -71,14 +74,6 @@ export function toolProfiles(value: unknown): ToolProfile[] | null {
     ? (value as ToolProfile[])
     : null;
 }
-/** The optional tool profiles and single tools every worker adds. */
-function validTools(value: Record<string, unknown>): boolean {
-  return (
-    (value['toolProfiles'] === undefined ||
-      toolProfiles(value['toolProfiles']) !== null) &&
-    (value['tools'] === undefined || toolNames(value['tools']))
-  );
-}
 function toolNames(value: unknown): boolean {
   return (
     Array.isArray(value) &&
@@ -89,9 +84,16 @@ function toolNames(value: unknown): boolean {
   );
 }
 export function parseWorkerPolicy(value: unknown): WorkerPolicy {
-  if (!record(value) || !validPolicy(value))
+  if (!record(value) || policyFault(value) !== null)
     throw new FleetUsageError(
       'Invalid mandatory worker policy. Workers require the locked toolchain, common services, unattended sudo, persistent T3 and no sleep.'
     );
-  return value as unknown as WorkerPolicy;
+  const exposure =
+    value['exposure'] === undefined
+      ? undefined
+      : parseExposure(value['exposure']);
+  return {
+    ...(value as unknown as WorkerPolicy),
+    ...(exposure ? { exposure } : {}),
+  };
 }

package/src/worker-readiness.ts

@@ -6,16 +6,25 @@ import { workerSelection } from '../bootstrap/worker-tools';
 import { browserCheck, dockerRunCheck } from './worker-capability-checks';
 import { gitChecks } from './git-identity';
 import type { Check, Runner } from './types';
-import { workerHostChecks } from './worker-host-checks';
+import { UBUNTU_WORKER } from './platform/ubuntu-worker';
 import { workerAccessChecks } from './worker-access-checks';
 import { record } from './config';
+import { workerExposure } from './exposure';
+import { exposureCheck } from './worker-exposure';
+import { notApplicable, offReason, workerFeatures } from './worker-features';
 
 export async function workerReadiness(
   run: Runner,
   nodeId = hostId(hostname()),
   root = process.cwd()
 ): Promise<Check[]> {
-  const host = await workerHostChecks(userInfo().username, run);
+  const features = workerFeatures(fleet, nodeId);
+  const host = await UBUNTU_WORKER.hostChecks(
+    userInfo().username,
+    run,
+    features,
+    fleet.workers.find((entry) => entry.id === nodeId)?.powerRecoveryNote
+  );
   const access = await workerAccessChecks(run, nodeId);
   const revision = await run(['git', 'rev-parse', 'HEAD']);
   const singleton = await run(
@@ -62,11 +71,14 @@ export async function workerReadiness(
     ...host,
     ...access,
     proxy,
-    await dockerRunCheck(run),
+    features.containers
+      ? await dockerRunCheck(run)
+      : notApplicable('worker:docker-run', offReason(features, 'docker')),
     await browserCheck(
       run,
       workerSelection(fleet, nodeId).profiles.includes('web')
     ),
+    await exposureCheck(workerExposure(fleet, nodeId), run),
     ...(await gitChecks(fleet.git, run)),
     ...(await workerCapacity(root, nodeId, run)),
   ];