@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/git-identity.ts

@@ -25,6 +25,7 @@ export async function convergeGitIdentity(
         git_name: git.name,
         git_email: git.email,
         git_home: home,
+        git_signing: git.signing ?? '',
       }),
     }
   );
@@ -33,7 +34,9 @@ export async function convergeGitIdentity(
       id: 'git:configuration',
       status: result.ok ? 'pass' : 'fail',
       summary: result.ok
-        ? 'The declared Git identity and the GitHub credential helper are written in ~/.config/git/config.'
+        ? git.signing
+          ? "The declared Git identity, the GitHub credential helper and commit signing with this node's own SSH key are written in ~/.config/git/config."
+          : 'The declared Git identity and the GitHub credential helper are written in ~/.config/git/config.'
         : 'GIT_CONFIGURATION_INCOMPLETE',
       ...(result.ok
         ? {}
@@ -152,9 +155,79 @@ export async function gitPushAuthCheck(
   };
 }
 
+/** The node's own commit signing key, made on the node by
+ * `ansible/github/git.yml`: an interface, like every path written to a
+ * machine.
+ */
+export function signingKey(home: string = homedir()): string {
+  return resolve(home, '.ssh/dev-fleet-git-signing');
+}
+
+/** What the signing probe signs: a fixed text, so nothing of the node's is. */
+const SIGNING_PROBE = 'dev-fleet signing probe\n';
+
+/** With `git.signing` declared, every commit and tag is signed with the
+ * node's own key: Git is configured to, and the key really signs. Without it
+ * there is no check.
+ */
+export async function gitSigningCheck(
+  git: GitIdentity | undefined,
+  run: Runner,
+  home: string = homedir()
+): Promise<Check | null> {
+  if (git?.signing !== 'ssh') return null;
+  const id = 'worker:git-signing';
+  const key = signingKey(home);
+  const format = await effective(run, 'gpg.format');
+  const signer = await effective(run, 'user.signingkey');
+  const always = await effective(run, 'commit.gpgsign');
+  if (format !== 'ssh' || signer !== `${key}.pub` || always !== 'true')
+    return {
+      id,
+      status: 'fail',
+      summary:
+        !format && !signer && !always
+          ? 'GIT_SIGNING_MISSING'
+          : 'GIT_SIGNING_OVERRIDDEN',
+      remedy: {
+        action:
+          'Rerun fleet-sync: it makes the node its own signing key and writes the signing settings in ~/.config/git/config. A ~/.gitconfig that sets gpg.format, user.signingkey or commit.gpgsign wins over it; remove those lines.',
+      },
+    };
+  const signed = await run(
+    ['ssh-keygen', '-Y', 'sign', '-n', 'git', '-f', key],
+    {
+      input: SIGNING_PROBE,
+      timeoutMs: 10_000,
+    }
+  );
+  if (!signed.ok || !signed.stdout.includes('-----BEGIN SSH SIGNATURE-----'))
+    return {
+      id,
+      status: 'fail',
+      summary: 'GIT_SIGNING_FAILED',
+      remedy: {
+        action:
+          "The node's signing key (~/.ssh/dev-fleet-git-signing) is missing or cannot sign without a prompt. Remove both halves of it and rerun fleet-sync, which makes a new one; then add the new public key to the GitHub account as a signing key.",
+      },
+    };
+  return {
+    id,
+    status: 'pass',
+    summary:
+      'Commits and tags on this node are signed with its own SSH key. GitHub shows them as verified once ~/.ssh/dev-fleet-git-signing.pub is a signing key of the account.',
+  };
+}
+
 export async function gitChecks(
   git: GitIdentity | undefined,
-  run: Runner
+  run: Runner,
+  home: string = homedir()
 ): Promise<Check[]> {
-  return [await gitIdentityCheck(git, run), await gitPushAuthCheck(git, run)];
+  const signing = await gitSigningCheck(git, run, home);
+  return [
+    await gitIdentityCheck(git, run),
+    await gitPushAuthCheck(git, run),
+    ...(signing ? [signing] : []),
+  ];
 }

package/src/ide/ide.ts

@@ -40,4 +40,9 @@ export interface WorkerContext {
    * time it is seen through a verified link.
    */
   remember?: (node: string, identity: string) => Promise<void>;
+  /** Pairs the manager's IDE with the worker's when the manager cannot
+   * reach it and a pairing was asked for (a replaced worker): the check
+   * that says how it went, or nothing when no pairing was asked for.
+   */
+  pair?: (node: FleetNode) => Promise<Check | undefined>;
 }

package/src/ide/t3.ts

@@ -78,7 +78,7 @@ export function t3Ide(inputs: T3Inputs = {}): Ide {
         await managerVersion(home(), context.workspace),
         await updatesCheck(
           inputs.policyPath ?? MACOS_POLICY,
-          `cd ${context.workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`
+          `cd ${context.workspace} && ${ansibleCommand().join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml`
         ),
       ];
     },
@@ -89,9 +89,19 @@ export function t3Ide(inputs: T3Inputs = {}): Ide {
       const restarted = context.earlier.some(
         (check) => check.summary === T3_RESTARTED_SUMMARY
       );
-      const linked = await verifyT3Linked(node, context.run, {
+      let linked = await verifyT3Linked(node, context.run, {
         reconnectMs: restarted ? 180_000 : 0,
       });
+      // A replaced worker: the manager pairs with its new environment.
+      const paired =
+        linked.status === 'fail' && context.pair
+          ? await context.pair(node)
+          : undefined;
+      if (paired) {
+        checks.push(paired);
+        if (paired.status === 'pass')
+          linked = await verifyT3Linked(node, context.run);
+      }
       checks.push(linked);
       if (
         linked.status !== 'fail' &&

package/src/input-hashes.ts

@@ -22,10 +22,16 @@ export async function fileHash(path: string): Promise<string | null> {
   }
 }
 
+/** A repository without a packages directory (dev-fleet itself) has an
+ * empty set of workspace manifests, which is known, not missing.
+ */
 export async function manifestHash(root: string): Promise<string | null> {
   try {
     const entries = await readdir(join(root, 'packages'), {
       withFileTypes: true,
+    }).catch((error: NodeJS.ErrnoException) => {
+      if (error.code === 'ENOENT') return [];
+      throw error;
     });
     const hashes: string[] = [];
     for (const entry of entries

package/src/machine/machine-class.ts

@@ -0,0 +1,59 @@
+import type { Runner } from '../types';
+
+/** What kind of machine a worker is. A virtual machine has no lid, no
+ * battery and no firmware its guest can read, and restarting it after a
+ * power loss is its provider's job: checks of those are not applicable to
+ * it. Containers and WSL2 are not workers at all (docs/support-matrix.md).
+ */
+export type MachineClass = 'physical' | 'virtual' | 'unsupported';
+
+/** What systemd-detect-virt prints for a container or WSL rather than a
+ * hypervisor. */
+const CONTAINERS = [
+  'container-other',
+  'docker',
+  'lxc',
+  'lxc-libvirt',
+  'openvz',
+  'podman',
+  'pouch',
+  'proot',
+  'rkt',
+  'systemd-nspawn',
+  'wsl',
+];
+
+/** The class of a machine from the probe's environment word (`machine`,
+ * `container` or `wsl`) and what systemd-detect-virt printed: `none` is bare
+ * metal, a hypervisor (`kvm`, `qemu`, `amazon`, `microsoft`, ...) a virtual
+ * machine. When it cannot tell (`unknown`, or nothing), the machine is
+ * taken as physical, so no check that applies to one is left out.
+ */
+export function machineClass(
+  environment: string,
+  virtualization: string
+): MachineClass {
+  if (environment !== 'machine' || CONTAINERS.includes(virtualization))
+    return 'unsupported';
+  return ['', 'none', 'unknown'].includes(virtualization)
+    ? 'physical'
+    : 'virtual';
+}
+
+/** The class of the machine this runs on, and the hypervisor, measured by
+ * systemd-detect-virt. The worker's own checks use it; the manager's probe
+ * records it for each worker before anything is installed. */
+export async function detectMachineClass(
+  run: Runner
+): Promise<{ machine: MachineClass; virtualization: string }> {
+  const detected = await run(['systemd-detect-virt']);
+  const word = detected.stdout.trim();
+  const virtualization = /^[a-z0-9-]{1,32}$/.test(word) ? word : 'unknown';
+  return {
+    machine: machineClass(
+      CONTAINERS.includes(virtualization) ? 'container' : 'machine',
+      virtualization
+    ),
+    virtualization,
+  };
+}

package/src/machine/platform.ts

@@ -1,10 +1,16 @@
 import platforms from '../../ansible/platforms.json';
-import { kernelArchitectures, ubuntuRelease } from '../platform/ubuntu-release';
+import { UBUNTU_WORKER, workerPlatform } from '../platform/ubuntu-worker';
+import { belowMinimum, MINIMUM_WORKER } from '../worker-sizing';
+import { machineClass } from './machine-class';
 
-/** One POSIX probe for every worker, four lines: family, release, CPU and
- * the environment it runs in (`wsl`, `container` or `machine`, a physical
- * or virtual machine of its own). Read-only, and it needs nothing installed.
- * `root` moves the files it reads, for a stand-in; workers use `/`.
+/** One POSIX probe for every worker, nine lines: family, release, CPU, the
+ * environment it runs in (`wsl`, `container` or `machine`, a physical or
+ * virtual machine of its own), the hypervisor systemd-detect-virt names
+ * (`none` on bare metal, `unknown` when it cannot tell), the CPU count, all
+ * memory in MiB, the size of the root filesystem in MiB, and its clock in
+ * seconds since the epoch, which the manager compares with its own.
+ * Read-only, and it needs nothing installed. `root` moves the files it
+ * reads, for a stand-in; workers use `/`.
  */
 export function platformProbe(root = ''): string {
   return `sh -c '${[
@@ -13,31 +19,54 @@ export function platformProbe(root = ''): string {
     `if grep -qiE "microsoft|wsl" ${root}/proc/sys/kernel/osrelease 2>/dev/null; then echo wsl`,
     `elif [ -e ${root}/.dockerenv ] || [ -e ${root}/run/.containerenv ] || systemd-detect-virt --container --quiet 2>/dev/null; then echo container`,
     'else echo machine; fi',
+    'v=$(systemd-detect-virt --vm 2>/dev/null)',
+    'echo "${v:-unknown}"',
+    'nproc 2>/dev/null || echo',
+    // Each size prints its own line, empty when unread, so the clock after
+    // them stays on the ninth.
+    `m=$(awk "/^MemTotal:/{print int(\\$2/1024)}" ${root}/proc/meminfo 2>/dev/null)`,
+    'echo "$m"',
+    `d=$(df -Pk ${root}/ 2>/dev/null | awk "NR==2{print int(\\$2/1024)}")`,
+    'echo "$d"',
+    'date -u +%s.%N',
   ].join('; ')}'`;
 }
 export const PLATFORM_PROBE = platformProbe();
 
 export interface PlatformBaseline {
-  id: 'ubuntu:release' | 'platform:environment';
+  id: 'ubuntu:release' | 'platform:environment' | 'platform:worker-size';
   supported: boolean;
   summary: string;
 }
 
-/** A worker is an accepted Ubuntu release on one of that release's CPUs. */
+/** A worker is an accepted release of a worker platform on one of that
+ * release's CPUs. The check keeps its ID and its refusal: Ubuntu is the only
+ * worker platform, and anything else is refused as not that baseline. */
 export function evaluatePlatform(probe: string): PlatformBaseline {
   const [family, version, cpu] = probe.split('\n').map((line) => line.trim());
-  const release = family === 'ubuntu' ? ubuntuRelease(version) : undefined;
+  const platform = workerPlatform(family);
+  const release = platform?.release(version);
   const supported =
-    release !== undefined && kernelArchitectures(release).includes(cpu ?? '');
+    platform !== undefined &&
+    release !== undefined &&
+    platform.kernelArchitectures(release).includes(cpu ?? '');
   return {
     id: 'ubuntu:release',
     supported,
     summary: supported
-      ? `Ubuntu ${release.release} and supported CPU architecture observed.`
-      : 'UBUNTU_BASELINE_MISMATCH',
+      ? `${platform.label} ${release.release} and supported CPU architecture observed.`
+      : (platform ?? UBUNTU_WORKER).mismatch,
   };
 }
 
+/** The clock the probe read, in seconds since the epoch, or undefined. */
+export function probedClock(probe: string): number | undefined {
+  const line = probe.split('\n')[8]?.trim() ?? '';
+  // GNU date prints nanoseconds; another date prints a literal N.
+  const match = /^(\d{9,})(\.\d+)?/.exec(line);
+  return match ? Number(match[1]! + (match[2] ?? '')) : undefined;
+}
+
 /** Environments the support matrix (ansible/platforms.json) refuses, by the
  * word the probe prints for them. */
 const REFUSED = new Map(
@@ -62,3 +91,53 @@ export function evaluateEnvironment(probe: string): PlatformBaseline {
       : `WORKER_ENVIRONMENT_UNSUPPORTED: ${refused ?? 'unknown environment'}`,
   };
 }
+
+const line = (probe: string, index: number): string =>
+  probe.split('\n')[index]?.trim() ?? '';
+
+/** The class of machine the probe found: physical, virtual or unsupported
+ * (a container or WSL2), recorded for every worker. A worker that cannot
+ * say which hypervisor it runs on is taken as physical, so nothing that
+ * applies to a physical machine is left out. */
+export function evaluateMachineClass(probe: string): {
+  id: 'platform:machine-class';
+  summary: string;
+} {
+  const environment = line(probe, 3);
+  const virtualization = line(probe, 4) || 'unknown';
+  const found = machineClass(environment, virtualization);
+  return {
+    id: 'platform:machine-class',
+    summary:
+      found === 'virtual'
+        ? `Virtual machine (${virtualization}): checks of firmware power recovery and of a lid do not apply.`
+        : found === 'physical'
+          ? `Physical machine (systemd-detect-virt reported ${virtualization}).`
+          : `Unsupported: ${environment || 'unknown environment'}.`,
+  };
+}
+
+const BELOW_MINIMUM = 'WORKER_SIZE_BELOW_MINIMUM';
+
+/** The machine against the stated minimum worker size (`sizing` in the
+ * Ubuntu policy), before anything is installed; null when the probe did not
+ * report a size. */
+export function evaluateSize(probe: string): PlatformBaseline | null {
+  const [cpus, memoryMiB, diskMiB] = [5, 6, 7].map((index) =>
+    Number(line(probe, index) || NaN)
+  );
+  if (![cpus, memoryMiB, diskMiB].every((value) => Number.isInteger(value)))
+    return null;
+  const lacks = belowMinimum({
+    cpus: cpus!,
+    memoryMiB: memoryMiB!,
+    diskMiB: diskMiB!,
+  });
+  return {
+    id: 'platform:worker-size',
+    supported: lacks.length === 0,
+    summary: lacks.length
+      ? `${BELOW_MINIMUM}: ${lacks.join(', ')}`
+      : `${cpus} CPUs, ${memoryMiB} MiB of memory and a ${diskMiB} MiB root filesystem: at least the minimum worker (${MINIMUM_WORKER.cpus} CPUs, ${MINIMUM_WORKER.memoryMiB} MiB, ${MINIMUM_WORKER.diskMiB} MiB).`,
+  };
+}

package/src/machine/state.ts

@@ -86,20 +86,39 @@ async function lockAbandoned(lock: string): Promise<boolean> {
   }
 }
 
-/** Removes an abandoned lock. The rename is atomic, so when several managers
- * race to recover it exactly one wins; the others simply retry the lock. */
-async function reclaimAbandonedLock(lock: string): Promise<boolean> {
+/** Removes an abandoned lock. Judging a lock abandoned and renaming it aside
+ * are two steps: between them another reclaimer can have replaced it with a
+ * live lock of its own. So the lock is judged again where it was put aside,
+ * where nobody else can change it, and put back when it turns out to be held.
+ * `judged` runs between the two steps; only the features pause there. */
+async function reclaimAbandonedLock(
+  lock: string,
+  judged: () => Promise<void>
+): Promise<boolean> {
   if (!(await lockAbandoned(lock))) return false;
+  await judged();
   const aside = `${lock}.abandoned-${randomUUID()}`;
   try {
     await rename(lock, aside);
   } catch {
     return true; // Someone else already recovered it; retry.
   }
+  if (!(await lockAbandoned(aside))) {
+    await rename(aside, lock).catch(() => undefined);
+    return false;
+  }
   await rm(aside, { recursive: true, force: true });
   return true;
 }
 
+/** Removes the lock only while it is still the one this run wrote. */
+async function releaseLock(lock: string, owner: string): Promise<void> {
+  const current = await readFile(join(lock, 'owner.json'), 'utf8').catch(
+    () => null
+  );
+  if (current === owner) await rm(lock, { recursive: true, force: true });
+}
+
 /** Bundles are written next to the lock and removed in a `finally`, which a
  * killed run never reaches. Holding the lock means none is in use. */
 async function sweepTransferBundles(root: string): Promise<void> {
@@ -113,7 +132,8 @@ async function sweepTransferBundles(root: string): Promise<void> {
  * cleanup. Never steals a lock from a still-running installer. */
 export async function withMachineLock<T>(
   root: string,
-  operation: () => Promise<T>
+  operation: () => Promise<T>,
+  judged: () => Promise<void> = async () => {}
 ): Promise<T> {
   await mkdir(root, { recursive: true, mode: 0o700 });
   const lock = join(root, 'sync.lock');
@@ -126,26 +146,28 @@ export async function withMachineLock<T>(
         throw new FleetUsageError(
           'Cannot create manager lock; inspect private state-directory permissions and free disk space.'
         );
-      if (attempt < 3 && (await reclaimAbandonedLock(lock))) continue;
+      if (attempt < 3 && (await reclaimAbandonedLock(lock, judged))) continue;
       throw new FleetLockError(
         'Fleet sync is locked. Inspect running installers before removing an abandoned sync.lock directory, then rerun.'
       );
     }
   }
+  const owner =
+    JSON.stringify({
+      version: 1,
+      pid: process.pid,
+      startedAt: new Date().toISOString(),
+      run: randomUUID(),
+    }) + '\n';
   try {
-    await writeFile(
-      join(lock, 'owner.json'),
-      JSON.stringify({
-        version: 1,
-        pid: process.pid,
-        startedAt: new Date().toISOString(),
-      }) + '\n',
-      { flag: 'wx', mode: 0o600 }
-    );
+    await writeFile(join(lock, 'owner.json'), owner, {
+      flag: 'wx',
+      mode: 0o600,
+    });
     await sweepTransferBundles(root);
     return await operation();
   } finally {
-    await rm(lock, { recursive: true });
+    await releaseLock(lock, owner);
   }
 }
 

package/src/machine/support-matrix.ts

@@ -1,6 +1,6 @@
 import macos from '../../ansible/macos/policy.json';
 import platforms from '../../ansible/platforms.json';
-import { UBUNTU_RELEASES } from '../platform/ubuntu-release';
+import { WORKER_PLATFORMS } from '../platform/ubuntu-worker';
 
 export type SupportStatus = 'supported' | 'best effort' | 'refused';
 export interface SupportRow {
@@ -25,14 +25,16 @@ export function supportMatrix(): SupportRow[] {
       status: 'supported',
       note: 'ansible/macos/policy.json',
     },
-    ...UBUNTU_RELEASES.map((entry): SupportRow => ({
-      role: 'worker',
-      platform: 'Ubuntu',
-      versions: `${entry.release} (${entry.codename})`,
-      architectures: entry.architectures.join(', '),
-      status: 'supported',
-      note: 'ansible/ubuntu/policy.json; a machine of its own, physical or virtual',
-    })),
+    ...WORKER_PLATFORMS.flatMap((platform) =>
+      platform.releases.map((entry): SupportRow => ({
+        role: 'worker',
+        platform: platform.label,
+        versions: `${entry.release} (${entry.codename})`,
+        architectures: entry.architectures.join(', '),
+        status: 'supported',
+        note: `${platform.policy}; a machine of its own, physical or virtual`,
+      }))
+    ),
     ...platforms.rows.map((row): SupportRow => ({
       role: row.role,
       platform: row.platform,

package/src/machine-health.ts

@@ -1,5 +1,6 @@
 import policy from '../ansible/ubuntu/policy.json';
 import type { DoctorCheck } from './doctor-check';
+import { threshold } from './worker-sizing';
 
 /** Machine health sample parsed from `net/remote-probe.sh` output. */
 export type HealthSample = {
@@ -8,6 +9,12 @@ export type HealthSample = {
   readonly memFreePct?: number;
   readonly swapPct?: number;
   readonly diskMb?: number;
+  /** The size of the home filesystem, in MiB. */
+  readonly diskTotalMb?: number;
+  /** All memory, in MiB. */
+  readonly memTotalMb?: number;
+  /** Agent sessions running now: T3's agent scopes. */
+  readonly agentSessions?: number;
   /** Free inodes on the home filesystem, as a percentage of all of them. */
   readonly inodesFreePct?: number;
   /** 1 when the clock is synchronised to a time source, 0 when not. */
@@ -24,6 +31,9 @@ const PROBE_KEYS = {
   memfree: 'memFreePct',
   swap: 'swapPct',
   diskmb: 'diskMb',
+  disktotalmb: 'diskTotalMb',
+  memtotalmb: 'memTotalMb',
+  agentsessions: 'agentSessions',
   inodefree: 'inodesFreePct',
   ntpsync: 'clockSynced',
   ntp: 'timeService',
@@ -52,6 +62,8 @@ const LABELS = {
   disk: 'Disk space',
   inodes: 'Free inodes',
 };
+/** The capacity checks of a machine, each reported as `worker:capacity:<id>`. */
+export const HEALTH_CHECK_IDS = Object.keys(LABELS);
 
 function check(
   id: keyof typeof LABELS,
@@ -92,17 +104,28 @@ export function memoryCheck(s: HealthSample): DoctorCheck | null {
   return check('memory', 'pass', what);
 }
 
-/** Free disk: <2 GB fails (VS Code server, git, worktrees break), <10 GB warns. */
-export function diskCheck(diskMb: number | undefined): DoctorCheck | null {
+/** Free disk, against lines that scale with the disk
+ * (`sizing.thresholds.capacityFail` and `capacityWarn` in policy.json): under
+ * the first fails (VS Code server, git, worktrees break), under the second
+ * warns. A disk of unknown size gets the upper bounds.
+ */
+export function diskCheck(
+  diskMb: number | undefined,
+  diskTotalMb?: number
+): DoctorCheck | null {
   if (diskMb === undefined) return null;
-  if (diskMb < 2048) {
+  const total = diskTotalMb ?? Number.MAX_SAFE_INTEGER;
+  const fail = threshold('capacityFail', total);
+  const warn = threshold('capacityWarn', total);
+  if (diskMb < fail) {
     return check(
       'disk',
       'fail',
       `almost full, ${diskMb} MB free. VS Code server, git and T3 worktrees will break.`
     );
   }
-  if (diskMb < 10_240) return check('disk', 'warn', `low, ${diskMb} MB free`);
+  if (diskMb < warn)
+    return check('disk', 'warn', `low, ${diskMb} MB free (under ${warn} MB)`);
   return check('disk', 'pass', `${Math.floor(diskMb / 1024)} GB free`);
 }
 
@@ -136,7 +159,7 @@ export function healthChecks(s: HealthSample, alias: string): DoctorCheck[] {
   return [
     loadCheck(s, alias),
     memoryCheck(s),
-    diskCheck(s.diskMb),
+    diskCheck(s.diskMb, s.diskTotalMb),
     inodesCheck(s.inodesFreePct),
   ].filter((c): c is DoctorCheck => c !== null);
 }