@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/input-hashes.ts

@@ -0,0 +1,73 @@
+import { createHash } from 'node:crypto';
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export const INPUT_FILES = [
+  'package.json',
+  'bun.lock',
+  'mise.toml',
+  'mise.lock',
+];
+export const FINGERPRINT_FILES = [
+  ...INPUT_FILES,
+  'workspace-manifests',
+  'mise-sidecars',
+];
+
+export async function fileHash(path: string): Promise<string | null> {
+  try {
+    return digest(await readFile(path));
+  } catch {
+    return null;
+  }
+}
+
+export async function manifestHash(root: string): Promise<string | null> {
+  try {
+    const entries = await readdir(join(root, 'packages'), {
+      withFileTypes: true,
+    });
+    const hashes: string[] = [];
+    for (const entry of entries
+      .filter((item) => item.isDirectory())
+      .sort((left, right) => left.name.localeCompare(right.name, 'en'))) {
+      hashes.push(
+        `${entry.name}:${await fileHash(join(root, 'packages', entry.name, 'package.json'))}`
+      );
+    }
+    return digest(hashes.join('\n'));
+  } catch {
+    return null;
+  }
+}
+
+export async function sidecarHash(root: string): Promise<string | null> {
+  try {
+    const files = await readdir(join(root, '.mise/locks'), {
+      recursive: true,
+      withFileTypes: true,
+    });
+    if (files.length > 4096 || files.some((entry) => entry.isSymbolicLink()))
+      return null;
+    const hashes: string[] = [];
+    const paths = files
+      .filter((entry) => entry.isFile())
+      .map((entry) => join(entry.parentPath, entry.name))
+      .sort();
+    for (const path of paths) {
+      const hash = await fileHash(path);
+      if (hash === null) return null;
+      const relative = path
+        .slice(join(root, '.mise/locks').length)
+        .replaceAll('\\', '/');
+      hashes.push(`${relative}:${hash}`);
+    }
+    return digest(hashes.join('\n'));
+  } catch {
+    return null;
+  }
+}
+
+function digest(value: string | Buffer): string {
+  return createHash('sha256').update(value).digest('hex');
+}

package/src/machine/local-source.ts

@@ -0,0 +1,52 @@
+import { isRevision } from './state';
+import { resolve } from 'node:path';
+import type { Runner } from '../types';
+import { fleet } from '../fleet-config';
+
+/** `https://host/owner/repo.git` and `git@host:owner/repo` name one repository. */
+export function sameRepository(remote: string, declared: string): boolean {
+  const name = (url: string) =>
+    url
+      .replace(/^(https:\/\/|ssh:\/\/git@|git@)/, '')
+      .replace(':', '/')
+      .replace(/\.git$/, '')
+      .toLowerCase();
+  return name(remote) === name(declared);
+}
+
+/** Use committed objects, never working-tree files, for an approved local branch. */
+export async function approvedLocalSource(
+  checkout: string,
+  revision: string,
+  run: Runner
+): Promise<string | null> {
+  const options = { cwd: checkout, timeoutMs: 10_000 };
+  const top = await run(['git', 'rev-parse', '--show-toplevel'], options);
+  if (!top.ok || resolve(top.stdout.trim()) !== resolve(checkout)) return null;
+  const origin = await run(['git', 'remote', 'get-url', 'origin'], options);
+  if (
+    !origin.ok ||
+    !sameRepository(origin.stdout.trim(), fleet.source.repository)
+  )
+    return null;
+  const commit = await run(
+    ['git', 'rev-parse', '--verify', `${revision}^{commit}`],
+    options
+  );
+  return commit.ok && commit.stdout.trim() === revision ? checkout : null;
+}
+
+export async function localCheckoutRevision(
+  checkout: string,
+  run: Runner
+): Promise<string | undefined> {
+  const result = await run(['git', 'rev-parse', 'HEAD'], {
+    cwd: checkout,
+    timeoutMs: 10_000,
+  });
+  const revision = result.stdout.trim();
+  if (!result.ok || !isRevision(revision)) return undefined;
+  return (await approvedLocalSource(checkout, revision, run))
+    ? revision
+    : undefined;
+}

package/src/machine/lock-error.ts

@@ -0,0 +1,2 @@
+import { FleetUsageError } from '../errors';
+export class FleetLockError extends FleetUsageError {}

package/src/machine/platform.ts

@@ -0,0 +1,44 @@
+import type { MachineDefinition } from '../definition';
+
+export type MachinePlatform = MachineDefinition['os'];
+
+/** One POSIX probe for every supported machine: family, release and CPU. */
+export const PLATFORM_PROBE =
+  'sh -c \'if [ "$(uname -s)" = Darwin ]; then printf "macos\\n%s\\n%s\\n" "$(sw_vers -productVersion)" "$(uname -m)"; else . /etc/os-release && printf "%s\\n%s\\n%s\\n" "$ID" "$VERSION_ID" "$(uname -m)"; fi\'';
+
+export interface PlatformBaseline {
+  id: 'ubuntu:release' | 'macos:release';
+  supported: boolean;
+  summary: string;
+}
+
+/** Ubuntu is the fully accepted worker platform; macOS is best effort. */
+export function evaluatePlatform(
+  os: MachinePlatform,
+  probe: string
+): PlatformBaseline {
+  if (os === 'macOS') {
+    const supported =
+      /^macos\n(1[4-9]|[2-9][0-9])(\.[0-9]+)*\n(arm64|x86_64)\s*$/.test(probe);
+    return {
+      id: 'macos:release',
+      supported,
+      summary: supported
+        ? 'macOS 14 or newer and a supported CPU architecture observed; macOS policy is best effort.'
+        : 'MACOS_BASELINE_MISMATCH',
+    };
+  }
+  const supported = /^ubuntu\n26\.04\n(x86_64|aarch64)\s*$/.test(probe);
+  return {
+    id: 'ubuntu:release',
+    supported,
+    summary: supported
+      ? 'Ubuntu 26.04 and supported CPU architecture observed.'
+      : 'UBUNTU_BASELINE_MISMATCH',
+  };
+}
+
+/** Windows machines stay declared but never receive manager convergence. */
+export function convergeable(node: MachineDefinition): boolean {
+  return node.os !== 'windows';
+}

package/src/machine/source.ts

@@ -0,0 +1,68 @@
+import { approvedLocalSource } from './local-source';
+import { join } from 'node:path';
+import { mkdir } from 'node:fs/promises';
+import type { Runner } from '../types';
+import { ensurePrivateRoot } from './state';
+import { fleet } from '../fleet-config';
+
+const repository = (): string => fleet.source.repository;
+
+/** Called after approval; local sources supply Git objects only, never dirty files. */
+export async function prepareSource(
+  root: string,
+  revision: string,
+  run: Runner,
+  checkout?: string
+): Promise<string | null> {
+  if (checkout) {
+    const local = await approvedLocalSource(checkout, revision, run);
+    if (local) return local;
+  }
+  return prepareRemoteSource(root, revision, run);
+}
+
+async function prepareRemoteSource(
+  root: string,
+  revision: string,
+  run: Runner
+): Promise<string | null> {
+  const source = join(root, 'source');
+  await ensurePrivateRoot(source);
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  let origin = await run(['git', 'remote', 'get-url', 'origin'], {
+    cwd: source,
+  });
+  if (!origin.ok) {
+    const cloned = await run(
+      [
+        'git',
+        'clone',
+        '--no-checkout',
+        '--origin',
+        'origin',
+        repository(),
+        source,
+      ],
+      { timeoutMs: 120_000 }
+    );
+    if (!cloned.ok) return null;
+    origin = await run(['git', 'remote', 'get-url', 'origin'], { cwd: source });
+  }
+  if (!origin.ok || origin.stdout.trim() !== repository()) return null;
+  let found = await run(
+    ['git', 'rev-parse', '--verify', `${revision}^{commit}`],
+    { cwd: source }
+  );
+  if (!found.ok || found.stdout.trim() !== revision) {
+    const fetch = await run(['git', 'fetch', '--no-tags', 'origin', revision], {
+      cwd: source,
+      timeoutMs: 120_000,
+    });
+    if (!fetch.ok) return null;
+    found = await run(
+      ['git', 'rev-parse', '--verify', `${revision}^{commit}`],
+      { cwd: source }
+    );
+  }
+  return found.ok && found.stdout.trim() === revision ? source : null;
+}

package/src/machine/state.ts

@@ -0,0 +1,173 @@
+import {
+  mkdir,
+  readFile,
+  readdir,
+  rename,
+  rm,
+  lstat,
+  stat,
+  writeFile,
+} from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { FleetUsageError } from '../errors';
+import { FleetLockError } from './lock-error';
+import { record } from '../config';
+
+export interface Enrollment {
+  version: 1;
+  id: string;
+  approvedRevision: string;
+  appliedRevision?: string;
+  previousRevision?: string;
+}
+
+export const isRevision = (value: unknown): value is string =>
+  typeof value === 'string' && /^[a-f0-9]{40}$/.test(value);
+
+/** The manager's enrollment from the file that held it before it moved into
+ * the fleet state (src/state/enrollment.ts). Nothing writes this file now.
+ */
+export async function readEnrollment(root: string): Promise<Enrollment | null> {
+  try {
+    const path = join(root, 'enrollment.json');
+    const info = await lstat(path);
+    if (!info.isFile() || info.size > 4096) throw new Error();
+    const enrollment = parseEnrollment(
+      JSON.parse(await readFile(path, 'utf8'))
+    );
+    if (!enrollment) throw new Error();
+    return enrollment;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw new FleetUsageError(
+      'Invalid enrollment state. Restore the private enrollment.json backup; do not enroll a replacement using another machine’s identity.'
+    );
+  }
+}
+
+/** A lock directory with no readable owner is only abandoned after this long,
+ * so a run that has just created it but not yet written `owner.json` is safe. */
+const OWNERLESS_LOCK_GRACE_MS = 60_000;
+
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM: the process exists but belongs to someone else.
+    return (error as NodeJS.ErrnoException).code !== 'ESRCH';
+  }
+}
+
+/** True only when the lock is provably abandoned: its recorded owner process
+ * no longer exists (killed, crashed, closed terminal, reboot), or it never
+ * recorded one and is older than the grace period. Anything uncertain is held. */
+async function lockAbandoned(lock: string): Promise<boolean> {
+  try {
+    const owner: unknown = JSON.parse(
+      await readFile(join(lock, 'owner.json'), 'utf8')
+    );
+    if (record(owner) && Number.isInteger(owner['pid']))
+      return !processAlive(owner['pid'] as number);
+  } catch {
+    // Missing or unreadable owner: fall through to the age check.
+  }
+  try {
+    return Date.now() - (await stat(lock)).mtimeMs > OWNERLESS_LOCK_GRACE_MS;
+  } catch {
+    return false;
+  }
+}
+
+/** Removes an abandoned lock. The rename is atomic, so when several managers
+ * race to recover it exactly one wins; the others simply retry the lock. */
+async function reclaimAbandonedLock(lock: string): Promise<boolean> {
+  if (!(await lockAbandoned(lock))) return false;
+  const aside = `${lock}.abandoned-${randomUUID()}`;
+  try {
+    await rename(lock, aside);
+  } catch {
+    return true; // Someone else already recovered it; retry.
+  }
+  await rm(aside, { recursive: true, force: true });
+  return true;
+}
+
+/** Bundles are written next to the lock and removed in a `finally`, which a
+ * killed run never reaches. Holding the lock means none is in use. */
+async function sweepTransferBundles(root: string): Promise<void> {
+  for (const name of await readdir(root).catch(() => [] as string[]))
+    if (/^transfer-[0-9a-f-]+\.bundle$/.test(name))
+      await rm(join(root, name), { force: true });
+}
+
+/** Fail closed while a sync is genuinely running; recover automatically from
+ * a lock whose owner process is gone so an interrupted run never needs manual
+ * cleanup. Never steals a lock from a still-running installer. */
+export async function withMachineLock<T>(
+  root: string,
+  operation: () => Promise<T>
+): Promise<T> {
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  const lock = join(root, 'sync.lock');
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await mkdir(lock, { mode: 0o700 });
+      break;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'EEXIST')
+        throw new FleetUsageError(
+          'Cannot create manager lock; inspect private state-directory permissions and free disk space.'
+        );
+      if (attempt < 3 && (await reclaimAbandonedLock(lock))) continue;
+      throw new FleetLockError(
+        'Fleet sync is locked. Inspect running installers before removing an abandoned sync.lock directory, then rerun.'
+      );
+    }
+  }
+  try {
+    await writeFile(
+      join(lock, 'owner.json'),
+      JSON.stringify({
+        version: 1,
+        pid: process.pid,
+        startedAt: new Date().toISOString(),
+      }) + '\n',
+      { flag: 'wx', mode: 0o600 }
+    );
+    await sweepTransferBundles(root);
+    return await operation();
+  } finally {
+    await rm(lock, { recursive: true });
+  }
+}
+
+export async function ensurePrivateRoot(root: string): Promise<void> {
+  for (let path = root; dirname(path) !== path; path = dirname(path)) {
+    try {
+      if ((await lstat(path)).isSymbolicLink())
+        throw new FleetUsageError(
+          'Fleet state paths cannot contain symbolic links.'
+        );
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+    }
+  }
+}
+
+/** A valid enrollment record, or null. */
+export function parseEnrollment(data: unknown): Enrollment | null {
+  if (!record(data) || data['version'] !== 1) return null;
+  if (
+    typeof data['id'] !== 'string' ||
+    !/^[a-z][a-z0-9-]{0,62}$/.test(data['id'])
+  )
+    return null;
+  return isRevision(data['approvedRevision']) &&
+    ['appliedRevision', 'previousRevision'].every(
+      (key) => data[key] === undefined || isRevision(data[key])
+    )
+    ? (data as unknown as Enrollment)
+    : null;
+}

package/src/machine-health.ts

@@ -0,0 +1,95 @@
+import type { DoctorCheck } from './doctor-check';
+
+/** Machine health sample parsed from `net/remote-probe.sh` output. */
+export type HealthSample = {
+  readonly ncpu?: number;
+  readonly load?: number;
+  readonly memFreePct?: number;
+  readonly swapPct?: number;
+  readonly diskMb?: number;
+};
+
+const PROBE_KEYS = {
+  ncpu: 'ncpu',
+  load: 'load',
+  memfree: 'memFreePct',
+  swap: 'swapPct',
+  diskmb: 'diskMb',
+} as const;
+
+function isProbeKey(key: string): key is keyof typeof PROBE_KEYS {
+  return Object.hasOwn(PROBE_KEYS, key);
+}
+
+/** Parse `key=value` probe lines; unknown keys and non-numbers are ignored. */
+export function parseProbe(output: string): HealthSample {
+  const sample: Record<string, number> = {};
+  for (const line of output.split(/\r?\n/)) {
+    const [key, value] = line.trim().split('=', 2);
+    if (key === undefined || value === undefined || !isProbeKey(key)) continue;
+    const n = Number(value);
+    if (value.length > 0 && Number.isFinite(n)) sample[PROBE_KEYS[key]] = n;
+  }
+  return sample;
+}
+
+const LABELS = { load: 'CPU load', memory: 'Memory', disk: 'Disk space' };
+
+function check(
+  id: keyof typeof LABELS,
+  severity: DoctorCheck['severity'],
+  detail: string,
+  fix?: string
+): DoctorCheck {
+  return { id, label: LABELS[id], severity, detail, ...(fix ? { fix } : {}) };
+}
+
+/** CPU load relative to core count: ≥2× per core fails, ≥1× warns. */
+export function loadCheck(s: HealthSample, alias: string): DoctorCheck | null {
+  if (s.ncpu === undefined || s.load === undefined || s.ncpu <= 0) return null;
+  const ratio = s.load / s.ncpu;
+  const what = `load ${s.load} on ${s.ncpu} cores (${ratio.toFixed(1)}x)`;
+  if (ratio >= 2) {
+    return check(
+      'load',
+      'fail',
+      `overloaded, ${what}. This makes SSH and T3 time out.`,
+      `Find the culprit: ssh ${alias} 'ps aux | sort -rk3 | head'`
+    );
+  }
+  if (ratio >= 1) return check('load', 'warn', `busy, ${what}`);
+  return check('load', 'pass', what);
+}
+
+/** RAM + swap pressure. */
+export function memoryCheck(s: HealthSample): DoctorCheck | null {
+  if (s.memFreePct === undefined || s.swapPct === undefined) return null;
+  const what = `${s.memFreePct}% RAM free, swap ${s.swapPct}% used`;
+  if (s.swapPct >= 90 && s.memFreePct < 15) {
+    return check('memory', 'fail', `exhausted, ${what}`);
+  }
+  if (s.memFreePct < 10 || s.swapPct >= 80) {
+    return check('memory', 'warn', `tight, ${what}`);
+  }
+  return check('memory', 'pass', what);
+}
+
+/** Free disk: <2 GB fails (VS Code server, git, worktrees break), <10 GB warns. */
+export function diskCheck(diskMb: number | undefined): DoctorCheck | null {
+  if (diskMb === undefined) return null;
+  if (diskMb < 2048) {
+    return check(
+      'disk',
+      'fail',
+      `almost full, ${diskMb} MB free. VS Code server, git and T3 worktrees will break.`
+    );
+  }
+  if (diskMb < 10_240) return check('disk', 'warn', `low, ${diskMb} MB free`);
+  return check('disk', 'pass', `${Math.floor(diskMb / 1024)} GB free`);
+}
+
+export function healthChecks(s: HealthSample, alias: string): DoctorCheck[] {
+  return [loadCheck(s, alias), memoryCheck(s), diskCheck(s.diskMb)].filter(
+    (c): c is DoctorCheck => c !== null
+  );
+}

package/src/managed-source.ts

@@ -0,0 +1,47 @@
+import { sshArgs } from './ssh';
+import { isRevision } from './machine/state';
+import type { Check, FleetNode, Runner } from './types';
+
+/** Audit the immutable deployed revision, never confuse it with a developer checkout. */
+export async function managedProbeTarget(
+  node: FleetNode,
+  revision: string | null,
+  run: Runner
+): Promise<{ node: FleetNode; checks: Check[] }> {
+  if (!node.required.includes('worker') || !isRevision(revision))
+    return { node, checks: [] };
+  const result = await run(
+    [
+      ...sshArgs(node),
+      `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf '%s' "$d"`,
+    ],
+    { timeoutMs: 15_000 }
+  );
+  if (!result.ok || !/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(result.stdout))
+    return {
+      node,
+      checks: [
+        {
+          id: 'worker:desired-source',
+          status: 'fail',
+          summary:
+            'The declared immutable worker revision has not been deployed. Developer checkout diagnostics follow separately.',
+          remedy: {
+            action:
+              'Run fleet-sync for this exact release/revision; developer branches and dirty work are never reset.',
+          },
+        },
+      ],
+    };
+  return {
+    node: { ...node, workspace: result.stdout },
+    checks: [
+      {
+        id: 'worker:desired-source',
+        status: 'pass',
+        summary:
+          'Auditing the deployed immutable revision, independently of the developer checkout.',
+      },
+    ],
+  };
+}

package/src/manager-failure.ts

@@ -0,0 +1,48 @@
+import { diagnostic } from './diagnostics';
+import type { FleetDefinition } from './definition';
+import type { ManagerReport } from './manager-sync';
+import type { Check } from './types';
+
+/** Even preflight failures account for every declared target, without discovery,
+ * authentication, filesystem mutations or accidental target substitution.
+ */
+export function managerFailure(
+  definition: FleetDefinition,
+  revision: string | null,
+  code: string,
+  action: string
+): ManagerReport {
+  const check: Check = {
+    id: 'manager:preflight',
+    status: 'fail',
+    summary: code,
+    remedy: { action },
+  };
+  return {
+    version: 3,
+    command: 'fleet-sync',
+    execution: 'manager-push',
+    ok: false,
+    applied: false,
+    desiredRevision: revision,
+    checks: [check],
+    candidates: 0,
+    dashboardUrl: null,
+    nodes: definition.workers.map((node) => ({
+      node: node.id,
+      diagnostics: [diagnostic(node.id, check)],
+      reachability: 'unknown',
+      desiredRevision: revision,
+      appliedRevision: null,
+      durationMs: 0,
+      checks: [
+        {
+          id: 'rollout:prerequisites',
+          status: 'skip',
+          summary: code,
+          remedy: { action },
+        },
+      ],
+    })),
+  };
+}