@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/machine/state.ts

@@ -0,0 +1,185 @@
+import {
+  mkdir,
+  readFile,
+  readdir,
+  rename,
+  rm,
+  lstat,
+  stat,
+  writeFile,
+} from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { FleetUsageError } from '../errors';
+import { FleetLockError } from './lock-error';
+import { record } from '../config';
+
+export interface Enrollment {
+  version: 1;
+  id: string;
+  approvedRevision: string;
+  appliedRevision?: string;
+  previousRevision?: string;
+}
+
+export const isRevision = (value: unknown): value is string =>
+  typeof value === 'string' && /^[a-f0-9]{40}$/.test(value);
+
+export async function readEnrollment(root: string): Promise<Enrollment | null> {
+  try {
+    const path = join(root, 'enrollment.json');
+    const info = await lstat(path);
+    if (!info.isFile() || info.size > 4096) throw new Error();
+    const data: unknown = JSON.parse(await readFile(path, 'utf8'));
+    if (!validEnrollment(data)) throw new Error();
+    return data as unknown as Enrollment;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw new FleetUsageError(
+      'Invalid enrollment state. Restore the private enrollment.json backup; do not enroll a replacement using another machine’s identity.'
+    );
+  }
+}
+
+export async function saveEnrollment(
+  root: string,
+  state: Enrollment
+): Promise<void> {
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  const path = join(root, 'enrollment.json');
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, JSON.stringify(state) + '\n', {
+      mode: 0o600,
+      flag: 'wx',
+    });
+    await rename(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+/** A lock directory with no readable owner is only abandoned after this long,
+ * so a run that has just created it but not yet written `owner.json` is safe. */
+const OWNERLESS_LOCK_GRACE_MS = 60_000;
+
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM: the process exists but belongs to someone else.
+    return (error as NodeJS.ErrnoException).code !== 'ESRCH';
+  }
+}
+
+/** True only when the lock is provably abandoned: its recorded owner process
+ * no longer exists (killed, crashed, closed terminal, reboot), or it never
+ * recorded one and is older than the grace period. Anything uncertain is held. */
+async function lockAbandoned(lock: string): Promise<boolean> {
+  try {
+    const owner: unknown = JSON.parse(
+      await readFile(join(lock, 'owner.json'), 'utf8')
+    );
+    if (record(owner) && Number.isInteger(owner['pid']))
+      return !processAlive(owner['pid'] as number);
+  } catch {
+    // Missing or unreadable owner: fall through to the age check.
+  }
+  try {
+    return Date.now() - (await stat(lock)).mtimeMs > OWNERLESS_LOCK_GRACE_MS;
+  } catch {
+    return false;
+  }
+}
+
+/** Removes an abandoned lock. The rename is atomic, so when several managers
+ * race to recover it exactly one wins; the others simply retry the lock. */
+async function reclaimAbandonedLock(lock: string): Promise<boolean> {
+  if (!(await lockAbandoned(lock))) return false;
+  const aside = `${lock}.abandoned-${randomUUID()}`;
+  try {
+    await rename(lock, aside);
+  } catch {
+    return true; // Someone else already recovered it; retry.
+  }
+  await rm(aside, { recursive: true, force: true });
+  return true;
+}
+
+/** Bundles are written next to the lock and removed in a `finally`, which a
+ * killed run never reaches. Holding the lock means none is in use. */
+async function sweepTransferBundles(root: string): Promise<void> {
+  for (const name of await readdir(root).catch(() => [] as string[]))
+    if (/^transfer-[0-9a-f-]+\.bundle$/.test(name))
+      await rm(join(root, name), { force: true });
+}
+
+/** Fail closed while a sync is genuinely running; recover automatically from
+ * a lock whose owner process is gone so an interrupted run never needs manual
+ * cleanup. Never steals a lock from a still-running installer. */
+export async function withMachineLock<T>(
+  root: string,
+  operation: () => Promise<T>
+): Promise<T> {
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  const lock = join(root, 'sync.lock');
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await mkdir(lock, { mode: 0o700 });
+      break;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'EEXIST')
+        throw new FleetUsageError(
+          'Cannot create manager lock; inspect private state-directory permissions and free disk space.'
+        );
+      if (attempt < 3 && (await reclaimAbandonedLock(lock))) continue;
+      throw new FleetLockError(
+        'Fleet sync is locked. Inspect running installers before removing an abandoned sync.lock directory, then rerun.'
+      );
+    }
+  }
+  try {
+    await writeFile(
+      join(lock, 'owner.json'),
+      JSON.stringify({
+        version: 1,
+        pid: process.pid,
+        startedAt: new Date().toISOString(),
+      }) + '\n',
+      { flag: 'wx', mode: 0o600 }
+    );
+    await sweepTransferBundles(root);
+    return await operation();
+  } finally {
+    await rm(lock, { recursive: true });
+  }
+}
+
+export async function ensurePrivateRoot(root: string): Promise<void> {
+  for (let path = root; dirname(path) !== path; path = dirname(path)) {
+    try {
+      if ((await lstat(path)).isSymbolicLink())
+        throw new FleetUsageError(
+          'Fleet state paths cannot contain symbolic links.'
+        );
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+    }
+  }
+}
+
+function validEnrollment(data: unknown): boolean {
+  if (!record(data) || data['version'] !== 1) return false;
+  if (
+    typeof data['id'] !== 'string' ||
+    !/^[a-z][a-z0-9-]{0,62}$/.test(data['id'])
+  )
+    return false;
+  return (
+    isRevision(data['approvedRevision']) &&
+    ['appliedRevision', 'previousRevision'].every(
+      (key) => data[key] === undefined || isRevision(data[key])
+    )
+  );
+}

package/src/machine-health.ts

@@ -0,0 +1,95 @@
+import type { DoctorCheck } from './doctor-check';
+
+/** Machine health sample parsed from `net/remote-probe.sh` output. */
+export type HealthSample = {
+  readonly ncpu?: number;
+  readonly load?: number;
+  readonly memFreePct?: number;
+  readonly swapPct?: number;
+  readonly diskMb?: number;
+};
+
+const PROBE_KEYS = {
+  ncpu: 'ncpu',
+  load: 'load',
+  memfree: 'memFreePct',
+  swap: 'swapPct',
+  diskmb: 'diskMb',
+} as const;
+
+function isProbeKey(key: string): key is keyof typeof PROBE_KEYS {
+  return Object.hasOwn(PROBE_KEYS, key);
+}
+
+/** Parse `key=value` probe lines; unknown keys and non-numbers are ignored. */
+export function parseProbe(output: string): HealthSample {
+  const sample: Record<string, number> = {};
+  for (const line of output.split(/\r?\n/)) {
+    const [key, value] = line.trim().split('=', 2);
+    if (key === undefined || value === undefined || !isProbeKey(key)) continue;
+    const n = Number(value);
+    if (value.length > 0 && Number.isFinite(n)) sample[PROBE_KEYS[key]] = n;
+  }
+  return sample;
+}
+
+const LABELS = { load: 'CPU load', memory: 'Memory', disk: 'Disk space' };
+
+function check(
+  id: keyof typeof LABELS,
+  severity: DoctorCheck['severity'],
+  detail: string,
+  fix?: string
+): DoctorCheck {
+  return { id, label: LABELS[id], severity, detail, ...(fix ? { fix } : {}) };
+}
+
+/** CPU load relative to core count: ≥2× per core fails, ≥1× warns. */
+export function loadCheck(s: HealthSample, alias: string): DoctorCheck | null {
+  if (s.ncpu === undefined || s.load === undefined || s.ncpu <= 0) return null;
+  const ratio = s.load / s.ncpu;
+  const what = `load ${s.load} on ${s.ncpu} cores (${ratio.toFixed(1)}x)`;
+  if (ratio >= 2) {
+    return check(
+      'load',
+      'fail',
+      `overloaded, ${what}. This makes SSH and T3 time out.`,
+      `Find the culprit: ssh ${alias} 'ps aux | sort -rk3 | head'`
+    );
+  }
+  if (ratio >= 1) return check('load', 'warn', `busy, ${what}`);
+  return check('load', 'pass', what);
+}
+
+/** RAM + swap pressure. */
+export function memoryCheck(s: HealthSample): DoctorCheck | null {
+  if (s.memFreePct === undefined || s.swapPct === undefined) return null;
+  const what = `${s.memFreePct}% RAM free, swap ${s.swapPct}% used`;
+  if (s.swapPct >= 90 && s.memFreePct < 15) {
+    return check('memory', 'fail', `exhausted, ${what}`);
+  }
+  if (s.memFreePct < 10 || s.swapPct >= 80) {
+    return check('memory', 'warn', `tight, ${what}`);
+  }
+  return check('memory', 'pass', what);
+}
+
+/** Free disk: <2 GB fails (VS Code server, git, worktrees break), <10 GB warns. */
+export function diskCheck(diskMb: number | undefined): DoctorCheck | null {
+  if (diskMb === undefined) return null;
+  if (diskMb < 2048) {
+    return check(
+      'disk',
+      'fail',
+      `almost full, ${diskMb} MB free. VS Code server, git and T3 worktrees will break.`
+    );
+  }
+  if (diskMb < 10_240) return check('disk', 'warn', `low, ${diskMb} MB free`);
+  return check('disk', 'pass', `${Math.floor(diskMb / 1024)} GB free`);
+}
+
+export function healthChecks(s: HealthSample, alias: string): DoctorCheck[] {
+  return [loadCheck(s, alias), memoryCheck(s), diskCheck(s.diskMb)].filter(
+    (c): c is DoctorCheck => c !== null
+  );
+}

package/src/managed-source.ts

@@ -0,0 +1,47 @@
+import { sshArgs } from './ssh';
+import { isRevision } from './machine/state';
+import type { Check, FleetNode, Runner } from './types';
+
+/** Audit the immutable deployed revision, never confuse it with a developer checkout. */
+export async function managedProbeTarget(
+  node: FleetNode,
+  revision: string | null,
+  run: Runner
+): Promise<{ node: FleetNode; checks: Check[] }> {
+  if (!node.required.includes('worker') || !isRevision(revision))
+    return { node, checks: [] };
+  const result = await run(
+    [
+      ...sshArgs(node),
+      `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf '%s' "$d"`,
+    ],
+    { timeoutMs: 15_000 }
+  );
+  if (!result.ok || !/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(result.stdout))
+    return {
+      node,
+      checks: [
+        {
+          id: 'worker:desired-source',
+          status: 'fail',
+          summary:
+            'The declared immutable worker revision has not been deployed. Developer checkout diagnostics follow separately.',
+          remedy: {
+            action:
+              'Run fleet-sync for this exact release/revision; developer branches and dirty work are never reset.',
+          },
+        },
+      ],
+    };
+  return {
+    node: { ...node, workspace: result.stdout },
+    checks: [
+      {
+        id: 'worker:desired-source',
+        status: 'pass',
+        summary:
+          'Auditing the deployed immutable revision, independently of the developer checkout.',
+      },
+    ],
+  };
+}

package/src/manager-failure.ts

@@ -0,0 +1,48 @@
+import { diagnostic } from './diagnostics';
+import type { FleetDefinition } from './definition';
+import type { ManagerReport } from './manager-sync';
+import type { Check } from './types';
+
+/** Even preflight failures account for every declared target, without discovery,
+ * authentication, filesystem mutations or accidental target substitution.
+ */
+export function managerFailure(
+  definition: FleetDefinition,
+  revision: string | null,
+  code: string,
+  action: string
+): ManagerReport {
+  const check: Check = {
+    id: 'manager:preflight',
+    status: 'fail',
+    summary: code,
+    remedy: { action },
+  };
+  return {
+    version: 3,
+    command: 'fleet-sync',
+    execution: 'manager-push',
+    ok: false,
+    applied: false,
+    desiredRevision: revision,
+    checks: [check],
+    candidates: 0,
+    dashboardUrl: null,
+    nodes: definition.workers.map((node) => ({
+      node: node.id,
+      diagnostics: [diagnostic(node.id, check)],
+      reachability: 'unknown',
+      desiredRevision: revision,
+      appliedRevision: null,
+      durationMs: 0,
+      checks: [
+        {
+          id: 'rollout:prerequisites',
+          status: 'skip',
+          summary: code,
+          remedy: { action },
+        },
+      ],
+    })),
+  };
+}