@crvouga/dev-fleet 0.3.0 → 0.4.0

package/src/worker-capability-checks.ts

@@ -0,0 +1,167 @@
+import policy from '../ansible/ubuntu/policy.json';
+import type { Check, CommandResult, Runner } from './types';
+
+/** What an agent does on a worker, tried the way it does it: start a
+ * container, build an image, launch a headless browser. Each check runs as
+ * the worker's user, in its session, and reports a code and a remedy, never
+ * what a command printed.
+ */
+
+function failure(id: string, summary: string, action: string): Check {
+  return { id, status: 'fail', summary, remedy: { action } };
+}
+
+/** Docker's own words for the causes an agent meets most. */
+function dockerFailure(result: CommandResult): {
+  code: string;
+  action: string;
+} {
+  const said = `${result.stdout}\n${result.stderr}`;
+  if (result.reason === 'timeout')
+    return {
+      code: 'DOCKER_RUN_TIMEOUT',
+      action:
+        'A container did not start and exit in time. Inspect the engine (systemctl status docker) and its disk; rerun fleet-sync.',
+    };
+  if (/permission denied[^\n]*docker\.sock/i.test(said))
+    return {
+      code: 'DOCKER_SOCKET_ACCESS_DENIED',
+      action:
+        "This session cannot use the Docker socket. fleet-sync adds the worker's user to the docker group; a session started before that keeps its old groups. Rerun fleet-sync, then restart the session (log out and in, or restart the T3 service).",
+    };
+  if (/cannot connect to the docker daemon/i.test(said))
+    return {
+      code: 'DOCKER_DAEMON_UNREACHABLE',
+      action:
+        'The Docker engine is not running. Rerun fleet-sync, which starts and enables docker.service.',
+    };
+  if (/no such image|pull access denied|not found/i.test(said))
+    return {
+      code: 'DOCKER_SMOKE_IMAGE_MISSING',
+      action:
+        'The pinned smoke-test image is not cached on this worker. Rerun fleet-sync, which pulls it once so this check works offline.',
+    };
+  if (/no space left on device/i.test(said))
+    return {
+      code: 'DOCKER_DISK_FULL',
+      action:
+        'The image store is full. Free space under /var/lib/docker (the disk GC prunes old images) and rerun fleet-sync.',
+    };
+  return {
+    code: 'DOCKER_RUN_FAILED',
+    action:
+      'Docker answers but cannot run a container here. Inspect cgroups and the engine log (journalctl -u docker), then rerun fleet-sync.',
+  };
+}
+
+/** Runs the pinned image offline, then builds a one-line image from it. */
+export async function dockerRunCheck(run: Runner): Promise<Check> {
+  const id = 'worker:docker-run';
+  const { image, timeoutSeconds } = policy.dockerSmokeTest;
+  const timeoutMs = timeoutSeconds * 1000;
+  const started = await run(
+    [
+      'docker',
+      'run',
+      '--rm',
+      '--network',
+      'none',
+      '--pull',
+      'never',
+      image,
+      'true',
+    ],
+    { timeoutMs }
+  );
+  if (!started.ok) {
+    const { code, action } = dockerFailure(started);
+    return failure(id, code, action);
+  }
+  const built = await run(
+    ['docker', 'build', '--quiet', '--pull=false', '--network', 'none', '-'],
+    { input: `FROM ${image}\nRUN true\n`, timeoutMs }
+  );
+  if (!built.ok) {
+    const { code, action } = dockerFailure(built);
+    return failure(
+      id,
+      code === 'DOCKER_RUN_FAILED' ? 'DOCKER_BUILD_FAILED' : code,
+      code === 'DOCKER_RUN_FAILED'
+        ? 'A container runs but an image does not build. Inspect Buildx (docker buildx ls) and the image store, then rerun fleet-sync.'
+        : action
+    );
+  }
+  return {
+    id,
+    status: 'pass',
+    summary:
+      'A container of the pinned image ran and a one-line image built, offline, in this session.',
+  };
+}
+
+/** Where `ansible/ubuntu/tasks/browser.yml` unpacks the pinned build. */
+export function headlessShell(arch: string = process.arch): string {
+  const { version, amd64, arm64 } = policy.browser.headlessShell;
+  const build = arch === 'arm64' ? arm64 : amd64;
+  const directory = build.archive.split('/').pop()!;
+  return `/opt/personal-fleet/browser/${version}/${directory}/chrome-headless-shell`;
+}
+
+/** A worker with the web profile launches the pinned headless Chromium,
+ * loads a blank page and exits, as a browser test would.
+ */
+export async function browserCheck(
+  run: Runner,
+  web: boolean,
+  arch: string = process.arch
+): Promise<Check> {
+  const id = 'worker:browser';
+  if (!web)
+    return {
+      id,
+      status: 'skip',
+      summary:
+        'This worker does not select the web tool profile, so no browser is installed for tests.',
+    };
+  const launched = await run(
+    [
+      headlessShell(arch),
+      '--headless',
+      '--no-sandbox',
+      '--disable-gpu',
+      '--dump-dom',
+      'about:blank',
+    ],
+    { timeoutMs: policy.browser.launchTimeoutSeconds * 1000 }
+  );
+  if (launched.ok && /<html/i.test(launched.stdout))
+    return {
+      id,
+      status: 'pass',
+      summary: `Headless Chromium ${policy.browser.headlessShell.version} launched, loaded a blank page and exited.`,
+    };
+  const said = `${launched.stdout}\n${launched.stderr}`;
+  if (launched.reason === 'missing')
+    return failure(
+      id,
+      'BROWSER_NOT_INSTALLED',
+      'The pinned headless Chromium is not installed. Rerun fleet-sync, which installs it on workers with the web profile.'
+    );
+  if (/error while loading shared libraries/i.test(said))
+    return failure(
+      id,
+      'BROWSER_LIBRARY_MISSING',
+      'A system library headless Chromium loads is missing. Rerun fleet-sync, which installs the pinned libraries of the web profile.'
+    );
+  if (launched.reason === 'timeout')
+    return failure(
+      id,
+      'BROWSER_LAUNCH_TIMEOUT',
+      'Headless Chromium did not exit in time. Check the worker for memory pressure, then rerun fleet-sync.'
+    );
+  return failure(
+    id,
+    'BROWSER_LAUNCH_FAILED',
+    'Headless Chromium did not load a blank page. Run it by hand with --dump-dom about:blank to see why, then rerun fleet-sync.'
+  );
+}

package/src/worker-capacity.ts

@@ -1,11 +1,14 @@
 import { join } from 'node:path';
+import { clockCheck } from './clock';
 import { healthChecks, parseProbe } from './machine-health';
 import type { Check, Runner } from './types';
 
 export async function workerCapacity(
   root: string,
   node: string,
-  run: Runner
+  run: Runner,
+  /** The manager's clock in seconds since the epoch, when it sent one. */
+  reference?: number
 ): Promise<Check[]> {
   const result = await run(['sh', join(root, 'ansible/net/remote-probe.sh')], {
     timeoutMs: 10_000,
@@ -25,10 +28,19 @@ export async function workerCapacity(
         summary: 'Required CPU, memory and disk evidence is unavailable.',
       },
     ];
-  return healthChecks(sample, node).map((check) => ({
-    id: `worker:capacity:${check.id}`,
-    status: check.severity,
-    summary: check.detail,
-    ...(check.fix ? { remedy: { action: check.fix } } : {}),
-  }));
+  return [
+    ...healthChecks(sample, node).map((check) => ({
+      id: `worker:capacity:${check.id}`,
+      status: check.severity,
+      summary: check.detail,
+      ...(check.fix ? { remedy: { action: check.fix } } : {}),
+    })),
+    clockCheck(
+      'worker:clock',
+      sample,
+      reference === undefined || sample.epoch === undefined
+        ? undefined
+        : sample.epoch - reference
+    ),
+  ];
 }

package/src/worker-host-checks.ts

@@ -95,6 +95,7 @@ export async function workerHostChecks(
       )
     );
   }
+  checks.push(await rebootPendingCheck(audit.rebootPending, run));
   checks.push(...(await recoveryChecks(run)));
   checks.push(...(await verifyUbuntuBoot(user, run)));
   checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
@@ -202,3 +203,34 @@ async function t3HealthCheck(run: Runner): Promise<Check> {
     'T3 local API answers with its environment identity within 10 seconds.'
   );
 }
+
+const DAY_SECONDS = 86_400;
+
+/** Kernel and library updates apply only after a reboot. A pending reboot
+ * warns with its age, and fails once it is older than `policy.json` allows.
+ */
+export async function rebootPendingCheck(
+  pending: boolean,
+  run: Runner,
+  now = Date.now()
+): Promise<Check> {
+  const id = 'worker:reboot-pending';
+  if (!pending) return { id, status: 'pass', summary: 'No reboot is pending.' };
+  const since = await run(['stat', '-c', '%Y', '/var/run/reboot-required']);
+  const seconds = Number(since.stdout.trim());
+  const days =
+    since.ok && /^\d+$/.test(since.stdout.trim())
+      ? Math.max(0, Math.floor((now / 1000 - seconds) / DAY_SECONDS))
+      : null;
+  const age = days === null ? '' : ` for ${days} days`;
+  return {
+    id,
+    status:
+      days !== null && days > policy.rebootPendingFailDays ? 'fail' : 'warn',
+    summary: `REBOOT_PENDING${age}: installed updates apply only after a reboot.`,
+    remedy: {
+      action:
+        'Reboot the worker when no agent turn is running (sudo systemctl reboot on the worker), then confirm it returns to ready with devfleet fleet-status --workers.',
+    },
+  };
+}

package/src/worker-policy.ts

@@ -1,6 +1,7 @@
 import { CAPABILITIES, record } from './config';
 import { FleetUsageError } from './errors';
 import type { Capability } from './types';
+import { TOOL_PROFILES, type ToolProfile } from '../bootstrap/worker-tools';
 
 export interface WorkerPolicy {
   requiredCapabilities: Capability[];
@@ -12,6 +13,10 @@ export interface WorkerPolicy {
   unattendedSudo: true;
   persistentT3: true;
   neverSleep: true;
+  /** Tool profiles every worker installs on top of the base tools. */
+  toolProfiles?: ToolProfile[];
+  /** Single tools pinned in dev-fleet's mise.toml every worker installs. */
+  tools?: string[];
 }
 const REQUIRED = [
   'github',
@@ -41,8 +46,11 @@ function validPolicy(value: Record<string, unknown>): boolean {
         'unattendedSudo',
         'persistentT3',
         'neverSleep',
+        'toolProfiles',
+        'tools',
       ].includes(key)
     ) &&
+    validTools(value) &&
     validCapabilities(value['requiredCapabilities']) &&
     typeof value['githubLogin'] === 'string' &&
     /^[a-zA-Z0-9-]{1,39}$/.test(value['githubLogin']) &&
@@ -55,6 +63,31 @@ function validPolicy(value: Record<string, unknown>): boolean {
     )
   );
 }
+/** A list of known tool profiles, each once; null when it is not one. */
+export function toolProfiles(value: unknown): ToolProfile[] | null {
+  return Array.isArray(value) &&
+    new Set(value).size === value.length &&
+    value.every((item) => TOOL_PROFILES.includes(item as ToolProfile))
+    ? (value as ToolProfile[])
+    : null;
+}
+/** The optional tool profiles and single tools every worker adds. */
+function validTools(value: Record<string, unknown>): boolean {
+  return (
+    (value['toolProfiles'] === undefined ||
+      toolProfiles(value['toolProfiles']) !== null) &&
+    (value['tools'] === undefined || toolNames(value['tools']))
+  );
+}
+function toolNames(value: unknown): boolean {
+  return (
+    Array.isArray(value) &&
+    new Set(value).size === value.length &&
+    value.every(
+      (item) => typeof item === 'string' && /^[a-z0-9@:/._-]{1,75}$/i.test(item)
+    )
+  );
+}
 export function parseWorkerPolicy(value: unknown): WorkerPolicy {
   if (!record(value) || !validPolicy(value))
     throw new FleetUsageError(

package/src/worker-readiness.ts

@@ -1,7 +1,10 @@
 import { workerCapacity } from './worker-capacity';
 import { userInfo, hostname } from 'node:os';
 import { hostId } from './topology';
-import { configPath } from './fleet-config';
+import { configPath, fleet } from './fleet-config';
+import { workerSelection } from '../bootstrap/worker-tools';
+import { browserCheck, dockerRunCheck } from './worker-capability-checks';
+import { gitChecks } from './git-identity';
 import type { Check, Runner } from './types';
 import { workerHostChecks } from './worker-host-checks';
 import { workerAccessChecks } from './worker-access-checks';
@@ -59,6 +62,12 @@ export async function workerReadiness(
     ...host,
     ...access,
     proxy,
+    await dockerRunCheck(run),
+    await browserCheck(
+      run,
+      workerSelection(fleet, nodeId).profiles.includes('web')
+    ),
+    ...(await gitChecks(fleet.git, run)),
     ...(await workerCapacity(root, nodeId, run)),
   ];
 }