@celilo/e2e 0.12.0 → 0.13.1

package/src/doctor.test.ts

@@ -0,0 +1,279 @@
+/**
+ * Recurrence gates for the cele2e preflight.
+ *
+ * Each of these conditions was, at least once, diagnosed as a product bug: the
+ * mistake was silent and the failure surfaced later and somewhere unrelated.
+ * Every test here breaks ONE condition deliberately and asserts the specific
+ * message the operator gets — per Rule 7.6, a gate nobody has seen fail is not
+ * a gate, so each assertion is written against the broken state, not the happy
+ * one.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import {
+  BAKED_MANAGEMENT_IMAGES,
+  type DockerfileBases,
+  type DoctorProbe,
+  EXPECTED_MANAGEMENT_CMD,
+  checkBaseImages,
+  checkDiskPressure,
+  checkManagementImage,
+  checkRunLock,
+  checkStaleContainers,
+  diagnose,
+  explainBuildFailure,
+  parseBaseImages,
+} from './doctor';
+import type { LockHolder, LockStatus } from './run-lock';
+
+/** A probe where everything is healthy; each test breaks exactly one thing. */
+function healthyProbe(overrides: Partial<DoctorProbe> = {}): DoctorProbe {
+  return {
+    imageExists: () => true,
+    imageCmd: () => [EXPECTED_MANAGEMENT_CMD],
+    staleContainers: () => [],
+    reclaimableImageBytes: () => 0,
+    ...overrides,
+  };
+}
+
+function holder(overrides: Partial<LockHolder> = {}): LockHolder {
+  return {
+    pid: 4242,
+    hostname: 'testhost',
+    session: 'branch (/work/tree)',
+    test: 'caddy-direct-internet',
+    runId: 'r1',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+    ...overrides,
+  };
+}
+
+function lockState(overrides: Partial<LockStatus> = {}): LockStatus {
+  return {
+    free: true,
+    holder: null,
+    heartbeatAgeMs: null,
+    suspect: false,
+    ownKept: false,
+    ...overrides,
+  };
+}
+
+describe('management image (problem 1: the bake fails as an SSH error)', () => {
+  test('a missing baked image fails preflight naming build-infra as the fix', () => {
+    const check = checkManagementImage(
+      healthyProbe({ imageExists: (ref) => ref !== 'celilo-e2e/management:latest' }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('celilo-e2e/management:latest');
+    expect(check.detail).toContain('cannot build');
+    expect(check.remedy).toBe('cele2e build-infra');
+  });
+
+  test('a bake that lost its Cmd fails, and says so as the SSH error it would become', () => {
+    // The bake container runs `sleep infinity`; docker commit persists that Cmd
+    // unless overridden. Without /startup.sh the ssh-keys volume is never
+    // populated and `machine add` dies with a connection error against a
+    // firewall IP — which reads as a network bug and is not one.
+    const check = checkManagementImage(healthyProbe({ imageCmd: () => ['sleep', 'infinity'] }));
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain(EXPECTED_MANAGEMENT_CMD);
+    expect(check.detail).toContain('ssh-keys');
+    expect(check.detail).toContain('Cannot connect to root@');
+    expect(check.remedy).toBe('cele2e build-infra');
+  });
+
+  test('both images present and correctly baked passes', () => {
+    const check = checkManagementImage(healthyProbe());
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain(String(BAKED_MANAGEMENT_IMAGES.length));
+  });
+});
+
+describe('base images (problem 3: a missing image reads as a network failure)', () => {
+  const dockerfiles: DockerfileBases[] = [
+    { derived: 'celilo-e2e/observer', bases: ['ubuntu:22.04'] },
+    { derived: 'celilo-e2e/pebble', bases: ['ghcr.io/letsencrypt/pebble:latest'] },
+  ];
+
+  test('a base image missing with its derived image ALSO missing fails, with the pull', () => {
+    // Post `docker image prune`: nothing local, so the build must go to a
+    // registry — the exact state that produced "failed to solve: ubuntu:22.04:
+    // net/http: TLS handshake timeout" 16 images into a rebuild.
+    const check = checkBaseImages(healthyProbe({ imageExists: () => false }), dockerfiles);
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('ubuntu:22.04');
+    expect(check.remedy).toContain('docker pull ubuntu:22.04');
+    expect(check.remedy).toContain('docker pull ghcr.io/letsencrypt/pebble:latest');
+  });
+
+  test('a base image missing but its derived image present passes — the build is cached', () => {
+    // This is the ordinary steady state on a working machine, and a check that
+    // failed here would refuse environments that run fine today.
+    const check = checkBaseImages(
+      healthyProbe({ imageExists: (ref) => ref.startsWith('celilo-e2e/') }),
+      dockerfiles,
+    );
+    expect(check.status).toBe('ok');
+  });
+
+  test('only the images that actually need fetching are named', () => {
+    const check = checkBaseImages(
+      healthyProbe({
+        imageExists: (ref) => ref !== 'ubuntu:22.04' && ref !== 'celilo-e2e/observer',
+      }),
+      dockerfiles,
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('ubuntu:22.04');
+    expect(check.detail).not.toContain('pebble');
+  });
+});
+
+describe('run-lock (problems 2 and 5)', () => {
+  test('a stale heartbeat with a live PID is SUSPECT, not merely busy', () => {
+    // The one hang PID-liveness cannot see: build-infra wedged at step [16/27],
+    // process alive and sleeping, heartbeat 1913s stale. Nothing surfaced it.
+    const check = checkRunLock(
+      lockState({
+        free: false,
+        holder: holder({ beatAt: Date.now() - 1_913_000 }),
+        heartbeatAgeMs: 1_913_000,
+        suspect: true,
+      }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('SUSPECT');
+    expect(check.detail).toContain('31m');
+    expect(check.detail).toContain('not progressing');
+    expect(check.remedy).toContain('4242');
+  });
+
+  test("this session's own kept lock is a warning, not contention", () => {
+    const check = checkRunLock(
+      lockState({
+        free: false,
+        holder: holder({ state: 'kept' }),
+        heartbeatAgeMs: 0,
+        ownKept: true,
+      }),
+    );
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('auto-releases');
+  });
+
+  test("another session's kept lock still fails", () => {
+    const check = checkRunLock(
+      lockState({ free: false, holder: holder({ state: 'kept' }), heartbeatAgeMs: 0 }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('another session');
+  });
+
+  test('a healthy busy lock reports the heartbeat age', () => {
+    const check = checkRunLock(
+      lockState({ free: false, holder: holder(), heartbeatAgeMs: 11_000 }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('11s ago');
+    expect(check.detail).not.toContain('SUSPECT');
+  });
+});
+
+describe('stale containers and disk pressure', () => {
+  test('leftover containers with no live run warn and point at `cele2e down`', () => {
+    const check = checkStaleContainers(
+      healthyProbe({ staleContainers: () => ['celilo-e2e-1_management', 'celilo-e2e-1_fw-main'] }),
+      lockState(),
+    );
+    expect(check.status).toBe('warn');
+    expect(check.remedy).toContain('cele2e down');
+  });
+
+  test('containers belonging to a live holder are not debris', () => {
+    const check = checkStaleContainers(
+      healthyProbe({ staleContainers: () => ['celilo-e2e-1_management'] }),
+      lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }),
+    );
+    expect(check.status).toBe('ok');
+  });
+
+  test('large reclaimable space steers away from `docker image prune`', () => {
+    const check = checkDiskPressure(healthyProbe({ reclaimableImageBytes: () => 40 * 1024 ** 3 }));
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('docker image prune');
+    expect(check.remedy).toContain('name=celilo-e2e');
+  });
+
+  test('disk pressure never blocks a run', () => {
+    const report = diagnose({
+      pkgDir: '/nonexistent',
+      probe: healthyProbe({ reclaimableImageBytes: () => 99 * 1024 ** 3 }),
+      lock: lockState(),
+    });
+    expect(report.checks.find((c) => c.name === 'disk')?.status).toBe('warn');
+    expect(report.ok).toBe(true);
+  });
+});
+
+describe('Dockerfile base parsing', () => {
+  test("ignores references to the Dockerfile's own build stages", () => {
+    const bases = parseBaseImages(
+      [
+        'FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS fetch',
+        'RUN apt-get update',
+        'FROM --platform=linux/amd64 eclipse-temurin:25-jre',
+        'COPY --from=fetch /out /out',
+      ].join('\n'),
+    );
+    expect(bases).toEqual(['debian:bookworm-slim', 'eclipse-temurin:25-jre']);
+  });
+
+  test('a plain single-stage Dockerfile yields its one base', () => {
+    expect(parseBaseImages('FROM ubuntu:22.04\nRUN echo hi\n')).toEqual(['ubuntu:22.04']);
+  });
+});
+
+describe('build failure translation', () => {
+  test('a "failed to solve: <image>" is explained as a missing local image', () => {
+    const text = explainBuildFailure(
+      'ERROR: failed to solve: ubuntu:22.04: failed to authorize: ... net/http: TLS handshake timeout',
+    );
+    expect(text).toContain('not in the local image store');
+    expect(text).toContain('docker pull ubuntu:22.04');
+  });
+
+  test('a genuine RUN-step failure is left alone', () => {
+    expect(
+      explainBuildFailure(
+        'ERROR: failed to solve: process "/bin/sh -c apt-get update" did not complete successfully: exit code: 100',
+      ),
+    ).toBe('');
+  });
+});
+
+describe('diagnose', () => {
+  test('any failing check makes the whole report not ok', () => {
+    const report = diagnose({
+      pkgDir: '/nonexistent',
+      probe: healthyProbe({ imageExists: () => false }),
+      lock: lockState(),
+    });
+    expect(report.ok).toBe(false);
+    expect(report.checks.find((c) => c.name === 'management-image')?.status).toBe('fail');
+  });
+
+  test('skipLock omits the lock check — run holds the lock itself', () => {
+    const report = diagnose({
+      pkgDir: '/nonexistent',
+      probe: healthyProbe(),
+      lock: lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }),
+      skipLock: true,
+    });
+    expect(report.checks.some((c) => c.name === 'run-lock')).toBe(false);
+    expect(report.ok).toBe(true);
+  });
+});

package/src/doctor.ts

@@ -0,0 +1,421 @@
+/**
+ * cele2e doctor — "is my environment sane?" as one mechanical answer.
+ *
+ * Every check here exists because its absence was diagnosed as a product bug
+ * first. They share a shape: the mistake is silent, and the failure surfaces
+ * later and somewhere unrelated. A missing bake shows up as an SSH error
+ * against a firewall IP; a pruned base image shows up as a TLS handshake
+ * timeout 16 images into a build. Each one is cheap to detect BEFORE a run
+ * starts, so `run` calls this as its implicit preflight and refuses rather
+ * than starting something that cannot succeed.
+ *
+ * Docker access is behind DoctorProbe (Rule 2.3) so every check is unit-
+ * testable with the condition deliberately broken — which is the point: a gate
+ * nobody has seen fail is not a gate (Rule 7.6).
+ */
+
+import { execFileSync } from 'node:child_process';
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import {
+  type LockHolder,
+  type LockStatus,
+  formatAge,
+  heartbeatAgeMs,
+  isSameSession,
+  isSuspect,
+  lockStatus,
+} from './run-lock';
+
+/**
+ * Images the per-test compose references by `image:` alone — it has no `build:`
+ * for them, so docker CANNOT produce them on demand. `cele2e build-infra` bakes
+ * both (install.sh, then `docker commit`). Missing → the run is doomed.
+ */
+export const BAKED_MANAGEMENT_IMAGES = [
+  'celilo-e2e/management:latest',
+  'celilo-e2e/management:vanilla',
+] as const;
+
+/**
+ * The bake commits a container that was started with `sleep infinity`, so it
+ * must override Cmd back to the management image's real entrypoint. When that
+ * override is lost the container comes up without ever running /startup.sh, the
+ * ssh-keys volume is never populated, and every machine the test adds fails
+ * with "Cannot connect to root@<ip> with provided SSH key" — an error that
+ * reads as a network or firewall bug and is neither (ce-um6).
+ */
+export const EXPECTED_MANAGEMENT_CMD = '/startup.sh';
+
+/** Warn above this much reclaimable image space — the pressure that makes people prune. */
+const RECLAIMABLE_WARN_BYTES = 20 * 1024 ** 3;
+
+export type CheckStatus = 'ok' | 'warn' | 'fail';
+
+export interface DoctorCheck {
+  /** Stable machine-readable id, e.g. 'management-image'. */
+  name: string;
+  status: CheckStatus;
+  /** What was actually found. */
+  detail: string;
+  /** The exact command that fixes it. Present whenever status is not 'ok'. */
+  remedy?: string;
+}
+
+export interface DoctorReport {
+  checks: DoctorCheck[];
+  /** True when no check failed. Warnings do not block a run. */
+  ok: boolean;
+}
+
+/** Every Docker fact doctor needs, injectable so the checks are testable. */
+export interface DoctorProbe {
+  /** Is this image present in the local store? An untagged ref means `:latest`. */
+  imageExists(ref: string): boolean;
+  /** The image's configured Cmd, or null when the image is absent. */
+  imageCmd(ref: string): string[] | null;
+  /** Names of leftover `celilo-e2e-*` containers. */
+  staleContainers(): string[];
+  /** Bytes docker reports as reclaimable image space. */
+  reclaimableImageBytes(): number;
+}
+
+// ─── Dockerfile parsing ──────────────────────────────────────────────
+
+/**
+ * External base images a Dockerfile pulls from, ignoring references to its own
+ * earlier build stages (`FROM x AS fetch` … `COPY --from=fetch`). Pure so the
+ * multi-stage and `--platform=` forms both stay covered by a unit test.
+ */
+export function parseBaseImages(dockerfile: string): string[] {
+  const stages = new Set<string>();
+  const bases: string[] = [];
+  for (const raw of dockerfile.split('\n')) {
+    const line = raw.trim();
+    const match = line.match(/^FROM\s+(.*)$/i);
+    if (!match) continue;
+    const tokens = match[1].split(/\s+/).filter((t) => !t.startsWith('--'));
+    const ref = tokens[0];
+    if (!ref) continue;
+    const asIdx = tokens.findIndex((t) => t.toLowerCase() === 'as');
+    if (asIdx >= 0 && tokens[asIdx + 1]) stages.add(tokens[asIdx + 1]);
+    if (!stages.has(ref) && !bases.includes(ref)) bases.push(ref);
+  }
+  return bases;
+}
+
+/** Map `docker/Dockerfile.observer` → `celilo-e2e/observer` (docker-compose-generator's imageTag). */
+function derivedImageTag(dockerfileName: string): string {
+  return `celilo-e2e/${dockerfileName.replace(/^Dockerfile\./, '')}`;
+}
+
+export interface DockerfileBases {
+  /** The `celilo-e2e/<name>` tag this Dockerfile produces. */
+  derived: string;
+  bases: string[];
+}
+
+/** Read every `docker/Dockerfile.*` and pair its produced tag with its base images. */
+export function readDockerfileBases(pkgDir: string): DockerfileBases[] {
+  const dir = join(pkgDir, 'docker');
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((f) => f.startsWith('Dockerfile.'))
+    .sort()
+    .map((f) => ({
+      derived: derivedImageTag(f),
+      bases: parseBaseImages(readFileSync(join(dir, f), 'utf-8')),
+    }));
+}
+
+// ─── Checks ──────────────────────────────────────────────────────────
+
+/**
+ * The run-lock, including the two states `status` used to render identically to
+ * a healthy one: a stale-heartbeat holder (suspect) and this session's own kept
+ * stack (friction, not contention — the next run clears it).
+ */
+export function checkRunLock(lock: LockStatus): DoctorCheck {
+  if (lock.free || !lock.holder) {
+    return { name: 'run-lock', status: 'ok', detail: 'free' };
+  }
+  const h = lock.holder;
+  const age = formatAge(lock.heartbeatAgeMs ?? 0);
+  if (lock.suspect) {
+    return {
+      name: 'run-lock',
+      status: 'fail',
+      detail: `SUSPECT — ${h.session} running ${h.test} (pid ${h.pid}) has not beaten its heartbeat for ${age}; the process is alive but is not progressing`,
+      remedy: `Inspect it (\`docker ps\`, \`kill -0 ${h.pid}\`); if it is wedged, kill ${h.pid} then \`cele2e down\``,
+    };
+  }
+  if (lock.ownKept) {
+    return {
+      name: 'run-lock',
+      status: 'warn',
+      detail: `held by THIS session's own kept stack (${h.test}) — the next run auto-releases it`,
+      remedy: 'cele2e release   (or `cele2e down` to also tear the stack down)',
+    };
+  }
+  if (h.state === 'kept') {
+    return {
+      name: 'run-lock',
+      status: 'fail',
+      detail: `held by another session's kept stack: ${h.session} (${h.test})`,
+      remedy: 'Wait for that session, or `cele2e release` if you know it is abandoned',
+    };
+  }
+  return {
+    name: 'run-lock',
+    status: 'fail',
+    detail: `busy — ${h.session} running ${h.test} (pid ${h.pid}), heartbeat ${age} ago`,
+    remedy: 'Poll with `cele2e status --json` (exit 0 = free, 3 = busy) and wait',
+  };
+}
+
+/** The baked management images exist AND still run /startup.sh. */
+export function checkManagementImage(probe: DoctorProbe): DoctorCheck {
+  const missing = BAKED_MANAGEMENT_IMAGES.filter((ref) => !probe.imageExists(ref));
+  if (missing.length > 0) {
+    return {
+      name: 'management-image',
+      status: 'fail',
+      detail: `baked image missing: ${missing.join(', ')} — the test compose references these by tag and cannot build them`,
+      remedy: 'cele2e build-infra',
+    };
+  }
+  const unbaked = BAKED_MANAGEMENT_IMAGES.filter(
+    (ref) => !(probe.imageCmd(ref) ?? []).includes(EXPECTED_MANAGEMENT_CMD),
+  );
+  if (unbaked.length > 0) {
+    return {
+      name: 'management-image',
+      status: 'fail',
+      detail: `${unbaked.join(', ')} does not run ${EXPECTED_MANAGEMENT_CMD} — the bake commit lost its Cmd, so the ssh-keys volume will stay empty and every \`machine add\` will fail with "Cannot connect to root@<ip> with provided SSH key"`,
+      remedy: 'cele2e build-infra',
+    };
+  }
+  return {
+    name: 'management-image',
+    status: 'ok',
+    detail: `${BAKED_MANAGEMENT_IMAGES.length} baked images present, both running ${EXPECTED_MANAGEMENT_CMD}`,
+  };
+}
+
+/**
+ * Base images the next build genuinely has to fetch.
+ *
+ * A base image absent from the local store is only a problem when the image
+ * built FROM it is also absent — with the derived tag present, buildkit serves
+ * the build from cache and never resolves the reference. Requiring every base
+ * unconditionally would refuse environments that work today, so the check is
+ * the conjunction: no derived image AND no base image means the build must go
+ * to a registry, which is where `failed to solve: ubuntu:22.04: net/http: TLS
+ * handshake timeout` comes from — 16 images deep, blamed on the network.
+ */
+export function checkBaseImages(probe: DoctorProbe, dockerfiles: DockerfileBases[]): DoctorCheck {
+  const needPull = new Set<string>();
+  for (const { derived, bases } of dockerfiles) {
+    if (probe.imageExists(derived)) continue;
+    for (const base of bases) {
+      if (!probe.imageExists(base)) needPull.add(base);
+    }
+  }
+  if (needPull.size === 0) {
+    return { name: 'base-images', status: 'ok', detail: 'every image the build needs is local' };
+  }
+  const refs = [...needPull].sort();
+  return {
+    name: 'base-images',
+    status: 'fail',
+    detail: `${refs.length} base image(s) absent locally and needed by an image that must be rebuilt: ${refs.join(', ')}`,
+    remedy: refs.map((r) => `docker pull ${r}`).join('\n           '),
+  };
+}
+
+/** Leftover containers from a crashed run, which start-of-run cleanup will wipe. */
+export function checkStaleContainers(probe: DoctorProbe, lock: LockStatus): DoctorCheck {
+  const names = probe.staleContainers();
+  if (names.length === 0) {
+    return {
+      name: 'stale-containers',
+      status: 'ok',
+      detail: 'no leftover celilo-e2e-* containers',
+    };
+  }
+  // With a live holder these are simply that run's containers, not debris.
+  if (!lock.free && !lock.ownKept) {
+    return {
+      name: 'stale-containers',
+      status: 'ok',
+      detail: `${names.length} celilo-e2e-* container(s) belonging to the current lock holder`,
+    };
+  }
+  return {
+    name: 'stale-containers',
+    status: 'warn',
+    detail: `${names.length} celilo-e2e-* container(s) with no live run: ${names.slice(0, 4).join(', ')}${names.length > 4 ? ', …' : ''}`,
+    remedy: 'cele2e down   (start-of-run cleanup also removes these)',
+  };
+}
+
+/**
+ * Reclaimable image space. A warning, never a failure — the point is to steer
+ * the reflex: under disk pressure people reach for `docker image prune`, which
+ * deletes the base images the next build needs. Pruning e2e containers by name
+ * frees space without costing a 27-image rebuild.
+ */
+export function checkDiskPressure(probe: DoctorProbe): DoctorCheck {
+  const bytes = probe.reclaimableImageBytes();
+  const gib = (bytes / 1024 ** 3).toFixed(1);
+  if (bytes < RECLAIMABLE_WARN_BYTES) {
+    return { name: 'disk', status: 'ok', detail: `${gib} GiB reclaimable image space` };
+  }
+  return {
+    name: 'disk',
+    status: 'warn',
+    detail: `${gib} GiB reclaimable image space — do NOT run \`docker image prune\`; it deletes the base images the next build needs and costs a full rebuild`,
+    remedy:
+      'docker ps -aq --filter name=celilo-e2e | xargs -r docker rm -f && docker volume prune -f',
+  };
+}
+
+/**
+ * Turn buildkit's `failed to solve: <ref>: … TLS handshake timeout` into the
+ * one sentence that identifies the cause.
+ *
+ * That message names the network because buildkit was, technically, on the
+ * network — but it only went there because the base image was not in the local
+ * store, which is what `docker image prune` (the reflex under disk pressure)
+ * removes. Reading it as a connectivity problem cost a full 27-image rebuild.
+ * checkBaseImages preflights for this, so reaching here means the image went
+ * missing mid-run; either way, say what to pull.
+ *
+ * Returns '' when the failure names a RUN step rather than an image reference —
+ * a genuine build error, which needs no translation.
+ */
+export function explainBuildFailure(stderr: string): string {
+  const ref = stderr.match(/failed to solve:\s*([^\s:"]+(?::[^\s:"]+)?)/)?.[1];
+  if (!ref || ref === 'process') return '';
+  return [
+    '',
+    '',
+    `This names "${ref}" — buildkit had to fetch it, which means it is not in the local image store.`,
+    `A \`docker image prune\` removes exactly these. Fix: docker pull ${ref}`,
+    'Then re-run. `cele2e doctor` checks for this before a run starts.',
+  ].join('\n');
+}
+
+// ─── Real probe ──────────────────────────────────────────────────────
+
+function docker(args: string[]): string {
+  return execFileSync('docker', args, {
+    encoding: 'utf-8',
+    stdio: ['ignore', 'pipe', 'ignore'],
+    timeout: 15_000,
+  }).trim();
+}
+
+export function createDockerProbe(): DoctorProbe {
+  // One `docker images` listing answers every existence question; per-image
+  // inspect calls would be ~40 process spawns on a preflight that must be fast.
+  let tags: Set<string> | null = null;
+  const knownTags = (): Set<string> => {
+    if (tags) return tags;
+    try {
+      tags = new Set(
+        docker(['images', '--format', '{{.Repository}}:{{.Tag}}'])
+          .split('\n')
+          .map((l) => l.trim())
+          .filter(Boolean),
+      );
+    } catch {
+      tags = new Set();
+    }
+    return tags;
+  };
+
+  return {
+    imageExists(ref) {
+      const withTag = ref.includes(':') ? ref : `${ref}:latest`;
+      return knownTags().has(withTag);
+    },
+    imageCmd(ref) {
+      try {
+        return JSON.parse(docker(['image', 'inspect', ref, '--format', '{{json .Config.Cmd}}']));
+      } catch {
+        return null;
+      }
+    },
+    staleContainers() {
+      try {
+        return docker(['ps', '-a', '--filter', 'name=celilo-e2e', '--format', '{{.Names}}'])
+          .split('\n')
+          .map((l) => l.trim())
+          .filter(Boolean);
+      } catch {
+        return [];
+      }
+    },
+    reclaimableImageBytes() {
+      try {
+        for (const line of docker(['system', 'df', '--format', '{{json .}}']).split('\n')) {
+          const row = JSON.parse(line) as { Type?: string; Reclaimable?: string };
+          if (row.Type !== 'Images') continue;
+          // Reclaimable looks like "12.3GB (48%)".
+          const m = row.Reclaimable?.match(/^([\d.]+)\s*([KMGT]?)B/i);
+          if (!m) return 0;
+          const scale = { '': 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }[
+            m[2].toUpperCase()
+          ];
+          return Number.parseFloat(m[1]) * (scale ?? 1);
+        }
+      } catch {}
+      return 0;
+    },
+  };
+}
+
+// ─── Report ──────────────────────────────────────────────────────────
+
+export interface DiagnoseOptions {
+  pkgDir: string;
+  probe?: DoctorProbe;
+  lock?: LockStatus;
+  /** Skip the run-lock check — `run` acquires the lock itself and reports contention its own way. */
+  skipLock?: boolean;
+}
+
+export function diagnose(options: DiagnoseOptions): DoctorReport {
+  const probe = options.probe ?? createDockerProbe();
+  const lock = options.lock ?? lockStatus();
+  const checks: DoctorCheck[] = [];
+  if (!options.skipLock) checks.push(checkRunLock(lock));
+  checks.push(checkManagementImage(probe));
+  checks.push(checkBaseImages(probe, readDockerfileBases(options.pkgDir)));
+  checks.push(checkStaleContainers(probe, lock));
+  checks.push(checkDiskPressure(probe));
+  return { checks, ok: !checks.some((c) => c.status === 'fail') };
+}
+
+const ICONS: Record<CheckStatus, string> = { ok: '✓', warn: '!', fail: '✗' };
+
+/** Render a report as the lines `doctor` prints and `run` prints on refusal. */
+export function formatReport(report: DoctorReport): string[] {
+  const lines: string[] = [];
+  for (const c of report.checks) {
+    lines.push(`  ${ICONS[c.status]} ${c.name.padEnd(18)} ${c.detail}`);
+    if (c.remedy) lines.push(`      fix: ${c.remedy}`);
+  }
+  return lines;
+}
+
+/** The holder line `status` prints, with heartbeat age always visible. */
+export function formatHolderLine(h: LockHolder): string {
+  const flags = [
+    isSuspect(h) ? 'SUSPECT' : null,
+    h.state === 'kept' && isSameSession(h) ? 'yours' : null,
+  ].filter(Boolean);
+  const suffix = flags.length ? ` [${flags.join(', ')}]` : '';
+  return `${h.session} — ${h.state} ${h.test} (pid ${h.pid}), heartbeat ${formatAge(heartbeatAgeMs(h))} ago${suffix}`;
+}