@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/monitoring/systems.ts

@@ -0,0 +1,138 @@
+import { record } from '../config';
+import type { Check } from '../types';
+import type { HubRequest } from './enroll';
+
+/** The hub keeps one system per worker, named by its fleet ID. These change
+ * that list when a worker is replaced or retired, and find the systems no
+ * declared worker matches. Only names and hosts are read; a record's other
+ * fields (tokens, users) are never returned.
+ */
+export interface HubSystem {
+  id: string;
+  name: string;
+  host: string;
+}
+
+function systems(value: unknown): HubSystem[] {
+  if (!record(value) || !Array.isArray(value['items']))
+    throw new Error('BESZEL_PROTOCOL');
+  return value['items'].flatMap((item) =>
+    record(item) &&
+    typeof item['id'] === 'string' &&
+    /^[a-z0-9]{15}$/.test(item['id']) &&
+    typeof item['name'] === 'string'
+      ? [
+          {
+            id: item['id'],
+            name: item['name'],
+            host: typeof item['host'] === 'string' ? item['host'] : '',
+          },
+        ]
+      : []
+  );
+}
+
+export async function listSystems(request: HubRequest): Promise<HubSystem[]> {
+  return systems(
+    await request(
+      '/api/collections/systems/records?perPage=500&fields=id,name,host'
+    )
+  );
+}
+
+async function named(request: HubRequest, node: string): Promise<HubSystem[]> {
+  return systems(
+    await request(
+      `/api/collections/systems/records?filter=${encodeURIComponent(`name="${node}"`)}&perPage=2&fields=id,name,host`
+    )
+  );
+}
+
+const check = (status: Check['status'], summary: string, action?: string) => ({
+  id: 'beszel:system',
+  status,
+  summary,
+  ...(action ? { remedy: { action } } : {}),
+});
+
+/** Points a replaced worker's system at the host it is reached by now, so
+ * the hub keeps one system for the ID and its history.
+ */
+export async function moveSystem(
+  request: HubRequest,
+  node: string,
+  host: string
+): Promise<Check> {
+  const found = await named(request, node);
+  if (found.length > 1)
+    return check(
+      'fail',
+      'BESZEL_DUPLICATE_NODE',
+      `The hub holds more than one system named ${node}. Delete the extra ones in the dashboard, then run the command again.`
+    );
+  if (!found[0])
+    return check(
+      'pass',
+      `The hub holds no system for ${node}; fleet-sync enrolls the new machine.`
+    );
+  if (found[0].host === host)
+    return check('pass', `The hub's system ${node} already watches ${host}.`);
+  await request(
+    `/api/collections/systems/records/${found[0].id}`,
+    { host },
+    'PATCH'
+  );
+  return check('pass', `The hub's system ${node} now watches ${host}.`);
+}
+
+/** Deletes a retired worker's system and its agent fingerprint. */
+export async function removeSystem(
+  request: HubRequest,
+  node: string
+): Promise<Check> {
+  const found = await named(request, node);
+  for (const system of found) {
+    const prints = await request(
+      `/api/collections/fingerprints/records?filter=${encodeURIComponent(`system="${system.id}"`)}&perPage=10&fields=id`
+    );
+    for (const print of record(prints) && Array.isArray(prints['items'])
+      ? prints['items']
+      : [])
+      if (record(print) && typeof print['id'] === 'string')
+        await request(
+          `/api/collections/fingerprints/records/${print['id']}`,
+          undefined,
+          'DELETE'
+        );
+    await request(
+      `/api/collections/systems/records/${system.id}`,
+      undefined,
+      'DELETE'
+    );
+  }
+  return check(
+    'pass',
+    found.length
+      ? `The hub no longer watches ${node}.`
+      : `The hub holds no system for ${node}.`
+  );
+}
+
+/** A system no declared worker (and not the manager) is named after: what a
+ * worker removed by hand, or destroyed outside dev-fleet, leaves behind.
+ */
+export function leftoverSystems(
+  found: HubSystem[],
+  declared: string[]
+): Check[] {
+  return found
+    .filter((system) => !declared.includes(system.name))
+    .map((system) => ({
+      id: 'beszel:leftover',
+      status: 'warn' as const,
+      summary: `The hub watches ${system.name}, which is no declared worker.`,
+      remedy: {
+        action: `If ${system.name} left the fleet, retire it (devfleet fleet-node-retire --id ${system.name} --apply --yes while it is still declared) or delete the system in the dashboard.`,
+      },
+    }));
+}

package/src/network-definition.ts

@@ -1,4 +1,8 @@
-import { type FleetDefinition, type MachineDefinition } from './definition';
+import {
+  sshTarget,
+  type FleetDefinition,
+  type MachineDefinition,
+} from './definition';
 import { record } from './config';
 import { FleetUsageError } from './errors';
 
@@ -19,8 +23,11 @@ export function networkProjection(
     node.network
       ? [
           {
-            alias: node.ssh,
-            ...node.network,
+            alias: sshTarget(node),
+            tailscaleName: node.network.tailscaleName,
+            user: node.network.user,
+            sshPort: node.network.sshPort,
+            t3Port: node.network.t3Port,
             os: node.os,
             identityFile: identities[node.id] ?? '',
           },
@@ -34,7 +41,9 @@ function compatible(
 ): node is MachineDefinition {
   return Boolean(
     node?.network &&
-    Object.entries(node.network).every(([key, value]) => host[key] === value) &&
+    Object.entries(node.network)
+      .filter(([key]) => key !== 'hostKey')
+      .every(([key, value]) => host[key] === value) &&
     node.os === host['os']
   );
 }

package/src/node-converge-linux.ts

@@ -18,17 +18,27 @@ import {
   convergeT3Service,
   runningVersionCheck,
   t3StartedAt,
+  t3Unit,
 } from './platform/t3-service';
+import {
+  featuresCheck,
+  presentParts,
+  workerFeatures,
+  type WorkerFeatures,
+} from './worker-features';
 import { userInfo } from 'node:os';
 import { resolve } from 'node:path';
 import { auditUbuntu } from './platform/ubuntu-audit';
+import { detectMachineClass } from './machine/machine-class';
 import { convergeGitIdentity } from './git-identity';
 import { workerSelection } from '../bootstrap/worker-tools';
+import { workerExposure } from './exposure';
 import {
   applyAgentProfiles,
   attempt,
   configureWorkstation,
 } from './node-converge-shared';
+import { loginServer } from './tailnet-control';
 import type { Check, Runner } from './types';
 
 /** Ubuntu worker convergence: Ansible owns OS policy, services and recovery; the
@@ -36,10 +46,11 @@ import type { Check, Runner } from './types';
  */
 export async function convergeLinux(
   root: string,
-  target: { node: string; revision: string },
+  target: { node: string; revision: string; restartBusy?: boolean },
   run: Runner
 ): Promise<Check[]> {
   const { node, revision } = target;
+  const restartBusy = target.restartBusy === true;
   const audit = await auditUbuntu(run);
   if (!audit.supported || !audit.passwordlessSudo)
     return [
@@ -58,12 +69,23 @@ export async function convergeLinux(
   const checks: Check[] = [];
   checks.push(
     ...(await attempt('tailscale:configuration', async () => [
-      await applyTailscale(root, run),
+      await applyTailscale(root, node, run),
     ]))
   );
+  const features = workerFeatures(fleet, node);
+  const before = await presentParts(run, userInfo().username, features);
   const { baseline, verified } = await baselineStep(root, run, node);
   checks.push(...baseline, ...verified);
-  checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
+  checks.push(
+    featuresCheck(
+      features,
+      await presentParts(run, userInfo().username, features),
+      before
+    )
+  );
+  checks.push(
+    ...(await t3AfterBaseline(root, { node, baseline, run, restartBusy }))
+  );
   checks.push(
     ...(await attempt('workstation:configuration', () =>
       configureWorkstation(root, run, node)
@@ -120,9 +142,10 @@ async function baselineStep(
   run: Runner,
   node: string
 ): Promise<{ baseline: Check[]; verified: Check[] }> {
+  const { machine } = await detectMachineClass(run);
   const measure = async () => [
     ...(await attempt('ubuntu:boot-policy', () =>
-      verifyUbuntuBoot(userInfo().username, run)
+      verifyUbuntuBoot(userInfo().username, run, machine)
     )),
     ...(await attempt('ubuntu:resource-safety', () =>
       verifyResourceSafety(userInfo().homedir, run)
@@ -151,6 +174,7 @@ async function applyBaseline(
   node: string
 ): Promise<{ cached: boolean; checks: Check[] }> {
   const user = userInfo();
+  const features = workerFeatures(fleet, node);
   const ubuntu = resolve(root, 'ansible/ubuntu');
   const result = await runAnsibleWithReceipt(
     run,
@@ -165,13 +189,18 @@ async function applyBaseline(
         development_home: user.homedir,
         // The web profile brings headless Chromium (tasks/browser.yml).
         worker_tool_profiles: workerSelection(fleet, node).profiles,
+        // What the firewall opens beyond the tailnet (tasks/firewall.yml).
+        worker_exposure: workerExposure(fleet, node).policy,
+        // What the declared features leave out (tasks/features.yml).
+        worker_containers: features.containers,
+        worker_session: features.session,
       }),
     },
     {
       name: 'ubuntu-baseline',
       sources: [ubuntu, resolve(root, 'ansible/github')],
       force,
-      verify: () => baselineServicesActive(run),
+      verify: () => baselineServicesActive(run, features),
     }
   );
   return {
@@ -191,14 +220,17 @@ async function applyBaseline(
 }
 
 /** Cheap probe for what the baseline owns: system services and user timers. */
-async function baselineServicesActive(run: Runner): Promise<boolean> {
+async function baselineServicesActive(
+  run: Runner,
+  features: WorkerFeatures
+): Promise<boolean> {
   const system = await run([
     'systemctl',
     'is-active',
     '--quiet',
     'ssh.service',
     'tailscaled.service',
-    'docker.service',
+    ...(features.containers ? ['docker.service'] : []),
   ]);
   if (!system.ok) return false;
   const user = await run([
@@ -212,12 +244,27 @@ async function baselineServicesActive(run: Runner): Promise<boolean> {
   return user.ok;
 }
 
-async function applyTailscale(root: string, run: Runner): Promise<Check> {
+/** A worker that was never logged in joins under its declared ID when the
+ * seed or the bootstrap left it a single-use key; the key carries the fleet's
+ * tag.
+ */
+async function applyTailscale(
+  root: string,
+  node: string,
+  run: Runner
+): Promise<Check> {
   const ubuntu = resolve(root, 'ansible/ubuntu');
   const tailscale = await runAnsibleWithReceipt(
     run,
     'tailscale.yml',
-    { cwd: ubuntu, timeoutMs: 300_000, input: '{}' },
+    {
+      cwd: ubuntu,
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        tailnet_hostname: node,
+        tailnet_login_server: loginServer(fleet.tailnet) ?? '',
+      }),
+    },
     {
       name: 'ubuntu-tailscale',
       sources: [ubuntu],
@@ -243,23 +290,30 @@ async function applyTailscale(root: string, run: Runner): Promise<Check> {
 async function convergeT3(
   root: string,
   node: string,
-  run: Runner
+  run: Runner,
+  restartBusy: boolean
 ): Promise<Check[]> {
   const artifact = managedT3();
-  const startedAt = await t3StartedAt(run);
-  const checks = await convergeT3Service(
-    artifact,
-    run,
-    async () => {
-      const staged = await stageT3Policy(root, artifact.path, run, node);
+  const { session } = workerFeatures(fleet, node);
+  const startedAt = await t3StartedAt(run, session);
+  const checks = await convergeT3Service(artifact, run, {
+    prepare: async () => {
+      const staged = await stageT3Policy(root, artifact.path, run, {
+        node,
+        restartBusy,
+      });
       return staged.status === 'pass' ? [] : [staged];
     },
-    () => installT3Desktop(root, run)
-  );
+    install: () =>
+      session === 'headless'
+        ? installT3Headless(root, artifact.path, run, restartBusy)
+        : installT3Desktop(root, run, restartBusy),
+    session,
+  });
   if (checks.every((check) => check.status === 'pass'))
     checks.push(
       await loadedT3Policy(artifact.path, run, {
-        restarted: startedAt !== (await t3StartedAt(run)),
+        restarted: startedAt !== (await t3StartedAt(run, session)),
       })
     );
   // Settings are watched by T3: providers, names and the profile apply
@@ -275,12 +329,16 @@ async function convergeT3(
   const serviceReady = checks.some(
     (check) => check.id === 't3:service' && check.status === 'pass'
   );
-  if (serviceReady) checks.push(await runningVersionCheck(run, userInfo().uid));
+  if (serviceReady)
+    checks.push(
+      await runningVersionCheck(run, userInfo().uid, Date.now(), session)
+    );
   if (serviceReady)
     checks.push(
       ...(await convergeT3Route(run, () => publishT3Route(root, run)))
     );
-  if (serviceReady) checks.push(...(await enableWorkerRecovery(root, run)));
+  if (serviceReady)
+    checks.push(...(await enableWorkerRecovery(root, run, session)));
   return checks;
 }
 
@@ -293,7 +351,11 @@ async function publishT3Route(root: string, run: Runner): Promise<boolean> {
   return result.ok;
 }
 
-async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
+async function installT3Desktop(
+  root: string,
+  run: Runner,
+  restartBusy: boolean
+): Promise<boolean> {
   const user = userInfo();
   const desktop = managedT3Desktop();
   const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -311,6 +373,7 @@ async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
         t3_desktop_url: desktop.url,
         t3_desktop_name: desktop.name,
         t3_desktop_sha256: desktop.sha256,
+        t3_restart_busy: restartBusy,
       }),
     },
     {
@@ -341,11 +404,55 @@ async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
   return result.ok;
 }
 
+/** A headless worker's T3: the pinned server the toolchain installed, as a
+ * lingering user service (t3-headless.yml). */
+async function installT3Headless(
+  root: string,
+  server: string,
+  run: Runner,
+  restartBusy: boolean
+): Promise<boolean> {
+  const user = userInfo();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const unit = t3Unit('headless');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 't3-headless.yml'),
+    {
+      cwd: ubuntu,
+      timeoutMs: 600_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+        t3_server_path: server,
+        t3_restart_busy: restartBusy,
+      }),
+    },
+    {
+      name: 'ubuntu-t3-headless',
+      sources: [ubuntu],
+      // A stopped or disabled server reruns the play, which starts it.
+      verify: async () =>
+        (await run(['systemctl', '--user', 'is-enabled', '--quiet', unit]))
+          .ok &&
+        (await run(['systemctl', '--user', 'is-active', '--quiet', unit])).ok,
+    }
+  );
+  return result.ok;
+}
+
 async function t3AfterBaseline(
   root: string,
-  context: { node: string; baseline: Check[]; run: Runner }
+  context: {
+    node: string;
+    baseline: Check[];
+    run: Runner;
+    restartBusy: boolean;
+  }
 ): Promise<Check[]> {
-  const { node, baseline, run } = context;
+  const { node, baseline, run, restartBusy } = context;
   if (baseline.some((check) => check.status !== 'pass'))
     return [
       {
@@ -358,12 +465,13 @@ async function t3AfterBaseline(
         },
       },
     ];
-  return attempt('t3:service', () => convergeT3(root, node, run));
+  return attempt('t3:service', () => convergeT3(root, node, run, restartBusy));
 }
 
 async function enableWorkerRecovery(
   root: string,
-  run: Runner
+  run: Runner,
+  session: WorkerFeatures['session']
 ): Promise<Check[]> {
   const user = userInfo();
   const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -378,6 +486,7 @@ async function enableWorkerRecovery(
         development_user: user.username,
         development_uid: user.uid,
         development_home: user.homedir,
+        ...(session === 'headless' ? { t3_unit: t3Unit(session) } : {}),
       }),
     },
     {
@@ -400,7 +509,7 @@ async function enableWorkerRecovery(
       id: 'worker:service-recovery',
       status: result.ok ? 'pass' : 'fail',
       summary: result.ok
-        ? 'T3 recovery enabled: a stopped desktop app is started and an unresponsive one restarted after three failed one-minute health checks.'
+        ? `T3 recovery enabled: a stopped ${session === 'headless' ? 'server' : 'desktop app'} is started and an unresponsive one restarted after three failed one-minute health checks.`
         : 'WORKER_RECOVERY_SETUP_FAILED',
     },
   ];

package/src/node-converge.ts

@@ -1,6 +1,7 @@
 import { runLockedCommand as runCommand } from './runner';
 import { isRevision } from './machine/state';
 import { convergeLinux } from './node-converge-linux';
+import { T3_RESTART_MODES } from './platform/t3-service';
 import type { Check } from './types';
 
 /** Internal remote entry, run on a worker only. The manager owns the operation
@@ -8,14 +9,23 @@ import type { Check } from './types';
  * prints child output. Workers are Linux.
  */
 async function main(): Promise<Check[]> {
-  const [node, revision] = process.argv.slice(2);
-  if (!node || !/^[a-z][a-z0-9-]{0,62}$/.test(node) || !isRevision(revision))
+  const [node, revision, t3Restart = 'keep-turns'] = process.argv.slice(2);
+  if (
+    !node ||
+    !/^[a-z][a-z0-9-]{0,62}$/.test(node) ||
+    !isRevision(revision) ||
+    !(T3_RESTART_MODES as readonly string[]).includes(t3Restart)
+  )
     throw new Error();
   const root = process.cwd();
   const head = await runCommand(['git', 'rev-parse', 'HEAD']);
   if (!head.ok || head.stdout.trim() !== revision) throw new Error();
   if (process.platform === 'linux')
-    return convergeLinux(root, { node, revision }, runCommand);
+    return convergeLinux(
+      root,
+      { node, revision, restartBusy: t3Restart === 'restart-busy' },
+      runCommand
+    );
   return [
     {
       id: 'platform:unsupported',

package/src/platform/ansible-receipt.ts

@@ -19,12 +19,18 @@ export interface Receipt {
   force?: boolean;
   /** Receipt directory; defaults to the user's fleet state. */
   directory?: string;
+  /** Run the playbook in check mode first, and apply only when that passes. */
+  checkFirst?: boolean;
+  /** Where the playbook's variables are written while it runs. */
+  scratch?: string;
   now?: number;
 }
 
 export interface ReceiptResult extends CommandResult {
   /** True when the playbook was skipped on an unchanged, verified receipt. */
   cached: boolean;
+  /** Which run the result is of: the check-mode run when it failed. */
+  stage?: 'check' | 'apply';
 }
 
 export function receiptDirectory(home = homedir()): string {
@@ -56,10 +62,19 @@ export async function runAnsibleWithReceipt(
   )
     return { ok: true, reason: 'exit', stdout: '', stderr: '', cached: true };
   await rm(path, { force: true }).catch(() => undefined);
-  const result = await runAnsible(run, playbook, options);
+  if (receipt.checkFirst) {
+    const checked = await runAnsible(
+      run,
+      playbook,
+      { ...options, check: true },
+      receipt.scratch
+    );
+    if (!checked.ok) return { ...checked, cached: false, stage: 'check' };
+  }
+  const result = await runAnsible(run, playbook, options, receipt.scratch);
   if (result.ok && key !== null)
     await record(directory, path, { version: 1, key, appliedAt: now });
-  return { ...result, cached: false };
+  return { ...result, cached: false, stage: 'apply' };
 }
 
 /** Best effort: a missing receipt only means the next run applies in full. */

package/src/platform/ansible.ts

@@ -30,6 +30,14 @@ export function ansibleCommand(
       ];
 }
 
+/** The package's Ansible configuration (ansible/ansible.cfg): how Ansible
+ * reaches a worker over SSH (timeouts, retries, pipelining, connection
+ * reuse) beside the options src/ssh-connection.ts gives every session.
+ */
+export function ansibleConfig(root = PACKAGE_ROOT): NodeJS.ProcessEnv {
+  return { ANSIBLE_CONFIG: join(root, 'ansible', 'ansible.cfg') };
+}
+
 /** Private fleet state on disk, never /tmp: a worker's /tmp is RAM-backed with a
  * per-user quota that agent caches can exhaust, and the baseline that bounds it
  * must still be able to run then.
@@ -45,7 +53,11 @@ export function variablesDirectory(home = homedir()): string {
 export async function runAnsible(
   run: Runner,
   playbook: string,
-  options: CommandOptions & { input: string },
+  options: CommandOptions & {
+    input: string;
+    /** Run in check mode: report what would change, change nothing. */
+    check?: boolean;
+  },
   scratch = variablesDirectory()
 ): Promise<CommandResult> {
   if (Buffer.byteLength(options.input) > 256 * 1024)
@@ -55,12 +67,36 @@ export async function runAnsible(
   const path = join(directory, 'variables.json');
   try {
     await writeFile(path, options.input, { mode: 0o600, flag: 'wx' });
-    const { input: _input, ...commandOptions } = options;
+    const { input: _input, check, ...commandOptions } = options;
     return await run(
-      [...ansibleCommand(), playbook, '--extra-vars', '@' + path],
-      commandOptions
+      [
+        ...ansibleCommand(),
+        playbook,
+        ...(check ? ['--check'] : []),
+        '--extra-vars',
+        '@' + path,
+      ],
+      { ...commandOptions, env: { ...ansibleConfig(), ...commandOptions.env } }
     );
   } finally {
     await rm(directory, { recursive: true, force: true });
   }
 }
+
+/** A playbook of the package, applied from the manager with these variables:
+ * one that addresses a worker over SSH itself (ansible/ubuntu/reboot.yml).
+ * Scenarios answer it in place of Ansible.
+ */
+export type PackagePlaybook = (
+  name: string,
+  variables: object,
+  timeoutMs: number
+) => Promise<CommandResult>;
+
+export function packagePlaybook(run: Runner): PackagePlaybook {
+  return (name, variables, timeoutMs) =>
+    runAnsible(run, join(PACKAGE_ROOT, 'ansible', name), {
+      input: JSON.stringify(variables),
+      timeoutMs,
+    });
+}