@celilo/e2e 0.11.3 → 0.13.0

package/src/docker-compose-generator.ts

@@ -7,6 +7,7 @@ import type { MachineSpec, NetworkConfig, TopologyPreset, Zone } from './types';
 import {
   ZONE_GATEWAYS,
   ZONE_SUBNETS,
+  externalWanIp,
   greenwaveRouterIp,
   internalResolverIp,
   zoneIp,
@@ -146,16 +147,50 @@ const PROTECTED_SUBNETS = (['dmz', 'app', 'secure', 'secure-mgmt'] as const)
   .map((zone) => ZONE_SUBNETS[zone])
   .join(' ');
 
-function defaultTopologyServices(): Record<string, unknown> {
+/**
+ * The zone legs fw-main actually holds, per topology — the SINGLE source of truth
+ * for "which networks is the customer firewall wired to".
+ *
+ * This exists because `deployFirewall`'s `opts.zones` was doing two jobs at once:
+ * saying which zones get `provided_networks` written to system config, AND
+ * standing in as a claim about the box's hardware. It was only ever the first.
+ * The compose file below is what decides the second, and roughly twenty call
+ * sites passed `['dmz']` while the generator wired four legs regardless — so
+ * every one of them under-declared, silently.
+ *
+ * Both the compose services and `deployFirewall` read this, so a test cannot
+ * under-declare by omission and a new leg cannot be added in one place only.
+ * `packages/e2e/tests/firewall-zone-legs.test.ts` holds them to each other.
+ */
+export type FirewallLeg = Zone | 'external';
+
+export function firewallZoneLegs(topology: TopologyPreset): FirewallLeg[] {
+  const segmented: FirewallLeg[] = ['internal', 'dmz', 'app', 'secure'];
+  return topology === 'direct-internet' ? [...segmented, 'external'] : segmented;
+}
+
+/**
+ * The compose `networks:` block for fw-main, derived from {@link firewallZoneLegs}
+ * so the wiring and the declaration cannot drift. `external` is the public leg and
+ * lives on the `isp-external` docker network rather than a zone-named one.
+ */
+function firewallNetworks(topology: TopologyPreset): Record<string, { ipv4_address: string }> {
+  const nets: Record<string, { ipv4_address: string }> = {};
+  for (const zone of firewallZoneLegs(topology)) {
+    if (zone === 'external') {
+      nets['isp-external'] = { ipv4_address: externalWanIp() };
+    } else {
+      nets[zone] = { ipv4_address: ZONE_GATEWAYS[zone] };
+    }
+  }
+  return nets;
+}
+
+function defaultTopologyServices(config: NetworkConfig): Record<string, unknown> {
   return {
     'fw-main': baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.firewall' },
-      networks: {
-        internal: { ipv4_address: ZONE_GATEWAYS.internal },
-        dmz: { ipv4_address: ZONE_GATEWAYS.dmz },
-        app: { ipv4_address: ZONE_GATEWAYS.app },
-        secure: { ipv4_address: ZONE_GATEWAYS.secure },
-      },
+      networks: firewallNetworks('default'),
       cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
       // A firewall may also terminate a VPN (the admin tunnel the `wireguard`
       // module owns). wireguard-go needs the tun device; the kernel module is
@@ -169,7 +204,7 @@ function defaultTopologyServices(): Record<string, unknown> {
       build: { context: '.', dockerfile: 'docker/Dockerfile.greenwave-sim' },
       networks: {
         internal: { ipv4_address: greenwaveRouterIp() },
-        'isp-external': { ipv4_address: '100.100.0.100' },
+        'isp-external': { ipv4_address: '203.0.113.100' },
       },
       cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
       // A firewall may also terminate a VPN (the admin tunnel the `wireguard`
@@ -178,6 +213,9 @@ function defaultTopologyServices(): Record<string, unknown> {
       devices: ['/dev/net/tun'],
       sysctls: ROUTER_SYSCTLS,
       volumes: ['ssh-keys:/ssh-keys:ro'],
+      // Which real ISP router this sim stands in for. The protocol is identical
+      // across the two; only the TR-181 vendor extension prefix differs.
+      environment: { ROUTER_VENDOR_PREFIX: config.routerVendorPrefix ?? 'X_GWS_' },
     }),
   };
 }
@@ -191,13 +229,7 @@ function directInternetTopologyServices(): Record<string, unknown> {
   return {
     'fw-main': baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.firewall' },
-      networks: {
-        internal: { ipv4_address: ZONE_GATEWAYS.internal },
-        dmz: { ipv4_address: ZONE_GATEWAYS.dmz },
-        app: { ipv4_address: ZONE_GATEWAYS.app },
-        secure: { ipv4_address: ZONE_GATEWAYS.secure },
-        'isp-external': { ipv4_address: '100.100.0.100' },
-      },
+      networks: firewallNetworks('direct-internet'),
       cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
       // A firewall may also terminate a VPN (the admin tunnel the `wireguard`
       // module owns). wireguard-go needs the tun device; the kernel module is
@@ -211,7 +243,10 @@ function directInternetTopologyServices(): Record<string, unknown> {
   };
 }
 
-const TOPOLOGY_SERVICES: Record<TopologyPreset, () => Record<string, unknown>> = {
+const TOPOLOGY_SERVICES: Record<
+  TopologyPreset,
+  (config: NetworkConfig) => Record<string, unknown>
+> = {
   default: defaultTopologyServices,
   'direct-internet': directInternetTopologyServices,
 };
@@ -296,7 +331,7 @@ export function generateSharedInfraYaml(): string {
     build: { context: '.', dockerfile: 'docker/Dockerfile.pebble' },
     networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PEBBLE } },
     cap_add: ['NET_ADMIN'],
-    command: '-config /config/pebble-config.json -dnsserver 100.100.0.1:53',
+    command: '-config /config/pebble-config.json -dnsserver 203.0.113.1:53',
     volumes: [
       './config/pebble/pebble-config.json:/config/pebble-config.json:ro',
       './config/pebble/pebble-tls.crt:/config/pebble-tls.crt:ro',
@@ -315,16 +350,49 @@ export function generateSharedInfraYaml(): string {
     volumes: getRegistryVolumes(),
   });
 
+  // OFF-FLEET recursive resolver — the rig's 1.1.1.1, and the vantage point
+  // celilo's `public_dns` check resolves from.
+  //
+  // A peer of comcast-resolver, not a replacement: the fleet is configured to
+  // use THAT one (`system init` writes dns.primary=203.0.113.1), and celilo
+  // refuses to accept a resolver it already asks as evidence about the public
+  // internet — a check that used the fleet's own resolver would pass whatever
+  // the internet sees, which is the celilo#626 blindness one layer up. So the
+  // simulation needs two independent public resolvers, exactly as the real
+  // world has your ISP's and Cloudflare's.
+  //
+  // Same image as comcast-resolver (identical unbound.conf → identical
+  // authoritative answers) with a different startup: this one sits ON
+  // internet-external with the authoritative servers, so it needs no route
+  // through fw-ext to reach them.
+  services['public-resolver'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.resolver' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PUBLIC_RESOLVER } },
+    cap_add: ['NET_ADMIN'],
+    command: '/public-resolver-startup.sh',
+  });
+
+  // IP echo service — mimics api.ipify.org. Reports the source address a
+  // request appears to come from, which for the customer fleet is the
+  // firewall's external address after SNAT. This is where the `public_dns`
+  // check's EXPECTATION comes from; comparing public DNS against the
+  // registrar's own response would be self-agreement.
+  services['ip-echo'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.ip-echo' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.IP_ECHO } },
+    cap_add: ['NET_ADMIN'],
+  });
+
   // External website prober simulator — mimics isitup.org's /api.json endpoint.
   // Sits on internet-external so probe requests exercise the real public-facing
   // path (target domain → DNS → ISP → firewall → Caddy). Uses comcast-resolver
-  // at 100.100.0.1 as DNS so it can recursively resolve the simulated public
+  // at 203.0.113.1 as DNS so it can recursively resolve the simulated public
   // zones (e.g. iamtheinternet.org).
   services.isitup = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.isitup' },
     networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.ISITUP } },
     cap_add: ['NET_ADMIN'],
-    dns: ['100.100.0.1'],
+    dns: ['203.0.113.1'],
   });
 
   // celilo.computer static site simulator — serves install.sh and the docs
@@ -336,7 +404,7 @@ export function generateSharedInfraYaml(): string {
     build: { context: '.', dockerfile: 'docker/Dockerfile.celilo-website-sim' },
     networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.WEBSITE } },
     cap_add: ['NET_ADMIN'],
-    dns: ['100.100.0.1'],
+    dns: ['203.0.113.1'],
   });
 
   // npm-compat registry simulator — serves @celilo/* tarballs to install.sh's
@@ -413,7 +481,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
     // Only when celilo-mgr lives off the internal LAN — keeps the default
     // topology's generated compose byte-identical for every existing test.
     ...(needsSecureMgmt ? { 'secure-mgmt': zoneNetworkDef('secure-mgmt') } : {}),
-    'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
+    'isp-external': networkDef('203.0.113.0/24', '203.0.113.250'),
     // internet-external is owned by shared infra; real-internet is per-test
     // (it just bridges fw-ext ↔ comcast-resolver, no shared service uses it).
     // real-internet uses Docker's default gateway (.1) because fw-ext's
@@ -532,7 +600,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
 
   // --- Topology-specific infrastructure (south of fw-ext) ---
   const topologyFn = TOPOLOGY_SERVICES[config.topology] || TOPOLOGY_SERVICES.default;
-  Object.assign(services, topologyFn());
+  Object.assign(services, topologyFn(config));
 
   // Give fw-main a leg on the control-plane network when celilo-mgr lives there,
   // so the management box is routed like any other segmented network rather than
@@ -549,7 +617,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   services['fw-ext'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.router' },
     networks: {
-      'isp-external': { ipv4_address: '100.100.0.101' },
+      'isp-external': { ipv4_address: '203.0.113.101' },
       'internet-external': { ipv4_address: '100.64.0.1' },
       'real-internet': { ipv4_address: '172.30.0.5' },
     },
@@ -560,7 +628,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   services['comcast-resolver'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.resolver' },
     networks: {
-      'isp-external': { ipv4_address: '100.100.0.1' },
+      'isp-external': { ipv4_address: '203.0.113.1' },
       'real-internet': { ipv4_address: '172.30.0.4' },
     },
     cap_add: ['NET_ADMIN'],
@@ -681,7 +749,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     dmz: zoneNetworkDef('dmz'),
     app: zoneNetworkDef('app'),
     secure: zoneNetworkDef('secure'),
-    'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
+    'isp-external': networkDef('203.0.113.0/24', '203.0.113.250'),
     'internet-external': networkDef('100.64.0.0/24', '100.64.0.250'),
     'real-internet': {
       driver: 'bridge',
@@ -715,7 +783,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   });
 
   const topologyFn = TOPOLOGY_SERVICES[config.topology] || TOPOLOGY_SERVICES.default;
-  Object.assign(services, topologyFn());
+  Object.assign(services, topologyFn(config));
 
   const legacyCaddyIp =
     config.dmzMachines.find((m) => m.name === 'caddy')?.ip ??
@@ -735,7 +803,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   services['fw-ext'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.router' },
     networks: {
-      'isp-external': { ipv4_address: '100.100.0.101' },
+      'isp-external': { ipv4_address: '203.0.113.101' },
       'internet-external': { ipv4_address: '100.64.0.1' },
       'real-internet': { ipv4_address: '172.30.0.5' },
     },
@@ -746,7 +814,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   services['comcast-resolver'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.resolver' },
     networks: {
-      'isp-external': { ipv4_address: '100.100.0.1' },
+      'isp-external': { ipv4_address: '203.0.113.1' },
       'real-internet': { ipv4_address: '172.30.0.4' },
     },
     cap_add: ['NET_ADMIN'],
@@ -808,7 +876,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     build: { context: '.', dockerfile: 'docker/Dockerfile.pebble' },
     networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PEBBLE } },
     cap_add: ['NET_ADMIN'],
-    command: '-config /config/pebble-config.json -dnsserver 100.100.0.1:53',
+    command: '-config /config/pebble-config.json -dnsserver 203.0.113.1:53',
     volumes: [
       './config/pebble/pebble-config.json:/config/pebble-config.json:ro',
       './config/pebble/pebble-tls.crt:/config/pebble-tls.crt:ro',

package/src/doctor.test.ts

@@ -0,0 +1,279 @@
+/**
+ * Recurrence gates for the cele2e preflight.
+ *
+ * Each of these conditions was, at least once, diagnosed as a product bug: the
+ * mistake was silent and the failure surfaced later and somewhere unrelated.
+ * Every test here breaks ONE condition deliberately and asserts the specific
+ * message the operator gets — per Rule 7.6, a gate nobody has seen fail is not
+ * a gate, so each assertion is written against the broken state, not the happy
+ * one.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import {
+  BAKED_MANAGEMENT_IMAGES,
+  type DockerfileBases,
+  type DoctorProbe,
+  EXPECTED_MANAGEMENT_CMD,
+  checkBaseImages,
+  checkDiskPressure,
+  checkManagementImage,
+  checkRunLock,
+  checkStaleContainers,
+  diagnose,
+  explainBuildFailure,
+  parseBaseImages,
+} from './doctor';
+import type { LockHolder, LockStatus } from './run-lock';
+
+/** A probe where everything is healthy; each test breaks exactly one thing. */
+function healthyProbe(overrides: Partial<DoctorProbe> = {}): DoctorProbe {
+  return {
+    imageExists: () => true,
+    imageCmd: () => [EXPECTED_MANAGEMENT_CMD],
+    staleContainers: () => [],
+    reclaimableImageBytes: () => 0,
+    ...overrides,
+  };
+}
+
+function holder(overrides: Partial<LockHolder> = {}): LockHolder {
+  return {
+    pid: 4242,
+    hostname: 'testhost',
+    session: 'branch (/work/tree)',
+    test: 'caddy-direct-internet',
+    runId: 'r1',
+    startedAt: new Date().toISOString(),
+    beatAt: Date.now(),
+    state: 'running',
+    ...overrides,
+  };
+}
+
+function lockState(overrides: Partial<LockStatus> = {}): LockStatus {
+  return {
+    free: true,
+    holder: null,
+    heartbeatAgeMs: null,
+    suspect: false,
+    ownKept: false,
+    ...overrides,
+  };
+}
+
+describe('management image (problem 1: the bake fails as an SSH error)', () => {
+  test('a missing baked image fails preflight naming build-infra as the fix', () => {
+    const check = checkManagementImage(
+      healthyProbe({ imageExists: (ref) => ref !== 'celilo-e2e/management:latest' }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('celilo-e2e/management:latest');
+    expect(check.detail).toContain('cannot build');
+    expect(check.remedy).toBe('cele2e build-infra');
+  });
+
+  test('a bake that lost its Cmd fails, and says so as the SSH error it would become', () => {
+    // The bake container runs `sleep infinity`; docker commit persists that Cmd
+    // unless overridden. Without /startup.sh the ssh-keys volume is never
+    // populated and `machine add` dies with a connection error against a
+    // firewall IP — which reads as a network bug and is not one.
+    const check = checkManagementImage(healthyProbe({ imageCmd: () => ['sleep', 'infinity'] }));
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain(EXPECTED_MANAGEMENT_CMD);
+    expect(check.detail).toContain('ssh-keys');
+    expect(check.detail).toContain('Cannot connect to root@');
+    expect(check.remedy).toBe('cele2e build-infra');
+  });
+
+  test('both images present and correctly baked passes', () => {
+    const check = checkManagementImage(healthyProbe());
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain(String(BAKED_MANAGEMENT_IMAGES.length));
+  });
+});
+
+describe('base images (problem 3: a missing image reads as a network failure)', () => {
+  const dockerfiles: DockerfileBases[] = [
+    { derived: 'celilo-e2e/observer', bases: ['ubuntu:22.04'] },
+    { derived: 'celilo-e2e/pebble', bases: ['ghcr.io/letsencrypt/pebble:latest'] },
+  ];
+
+  test('a base image missing with its derived image ALSO missing fails, with the pull', () => {
+    // Post `docker image prune`: nothing local, so the build must go to a
+    // registry — the exact state that produced "failed to solve: ubuntu:22.04:
+    // net/http: TLS handshake timeout" 16 images into a rebuild.
+    const check = checkBaseImages(healthyProbe({ imageExists: () => false }), dockerfiles);
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('ubuntu:22.04');
+    expect(check.remedy).toContain('docker pull ubuntu:22.04');
+    expect(check.remedy).toContain('docker pull ghcr.io/letsencrypt/pebble:latest');
+  });
+
+  test('a base image missing but its derived image present passes — the build is cached', () => {
+    // This is the ordinary steady state on a working machine, and a check that
+    // failed here would refuse environments that run fine today.
+    const check = checkBaseImages(
+      healthyProbe({ imageExists: (ref) => ref.startsWith('celilo-e2e/') }),
+      dockerfiles,
+    );
+    expect(check.status).toBe('ok');
+  });
+
+  test('only the images that actually need fetching are named', () => {
+    const check = checkBaseImages(
+      healthyProbe({
+        imageExists: (ref) => ref !== 'ubuntu:22.04' && ref !== 'celilo-e2e/observer',
+      }),
+      dockerfiles,
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('ubuntu:22.04');
+    expect(check.detail).not.toContain('pebble');
+  });
+});
+
+describe('run-lock (problems 2 and 5)', () => {
+  test('a stale heartbeat with a live PID is SUSPECT, not merely busy', () => {
+    // The one hang PID-liveness cannot see: build-infra wedged at step [16/27],
+    // process alive and sleeping, heartbeat 1913s stale. Nothing surfaced it.
+    const check = checkRunLock(
+      lockState({
+        free: false,
+        holder: holder({ beatAt: Date.now() - 1_913_000 }),
+        heartbeatAgeMs: 1_913_000,
+        suspect: true,
+      }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('SUSPECT');
+    expect(check.detail).toContain('31m');
+    expect(check.detail).toContain('not progressing');
+    expect(check.remedy).toContain('4242');
+  });
+
+  test("this session's own kept lock is a warning, not contention", () => {
+    const check = checkRunLock(
+      lockState({
+        free: false,
+        holder: holder({ state: 'kept' }),
+        heartbeatAgeMs: 0,
+        ownKept: true,
+      }),
+    );
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('auto-releases');
+  });
+
+  test("another session's kept lock still fails", () => {
+    const check = checkRunLock(
+      lockState({ free: false, holder: holder({ state: 'kept' }), heartbeatAgeMs: 0 }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('another session');
+  });
+
+  test('a healthy busy lock reports the heartbeat age', () => {
+    const check = checkRunLock(
+      lockState({ free: false, holder: holder(), heartbeatAgeMs: 11_000 }),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('11s ago');
+    expect(check.detail).not.toContain('SUSPECT');
+  });
+});
+
+describe('stale containers and disk pressure', () => {
+  test('leftover containers with no live run warn and point at `cele2e down`', () => {
+    const check = checkStaleContainers(
+      healthyProbe({ staleContainers: () => ['celilo-e2e-1_management', 'celilo-e2e-1_fw-main'] }),
+      lockState(),
+    );
+    expect(check.status).toBe('warn');
+    expect(check.remedy).toContain('cele2e down');
+  });
+
+  test('containers belonging to a live holder are not debris', () => {
+    const check = checkStaleContainers(
+      healthyProbe({ staleContainers: () => ['celilo-e2e-1_management'] }),
+      lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }),
+    );
+    expect(check.status).toBe('ok');
+  });
+
+  test('large reclaimable space steers away from `docker image prune`', () => {
+    const check = checkDiskPressure(healthyProbe({ reclaimableImageBytes: () => 40 * 1024 ** 3 }));
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('docker image prune');
+    expect(check.remedy).toContain('name=celilo-e2e');
+  });
+
+  test('disk pressure never blocks a run', () => {
+    const report = diagnose({
+      pkgDir: '/nonexistent',
+      probe: healthyProbe({ reclaimableImageBytes: () => 99 * 1024 ** 3 }),
+      lock: lockState(),
+    });
+    expect(report.checks.find((c) => c.name === 'disk')?.status).toBe('warn');
+    expect(report.ok).toBe(true);
+  });
+});
+
+describe('Dockerfile base parsing', () => {
+  test("ignores references to the Dockerfile's own build stages", () => {
+    const bases = parseBaseImages(
+      [
+        'FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS fetch',
+        'RUN apt-get update',
+        'FROM --platform=linux/amd64 eclipse-temurin:25-jre',
+        'COPY --from=fetch /out /out',
+      ].join('\n'),
+    );
+    expect(bases).toEqual(['debian:bookworm-slim', 'eclipse-temurin:25-jre']);
+  });
+
+  test('a plain single-stage Dockerfile yields its one base', () => {
+    expect(parseBaseImages('FROM ubuntu:22.04\nRUN echo hi\n')).toEqual(['ubuntu:22.04']);
+  });
+});
+
+describe('build failure translation', () => {
+  test('a "failed to solve: <image>" is explained as a missing local image', () => {
+    const text = explainBuildFailure(
+      'ERROR: failed to solve: ubuntu:22.04: failed to authorize: ... net/http: TLS handshake timeout',
+    );
+    expect(text).toContain('not in the local image store');
+    expect(text).toContain('docker pull ubuntu:22.04');
+  });
+
+  test('a genuine RUN-step failure is left alone', () => {
+    expect(
+      explainBuildFailure(
+        'ERROR: failed to solve: process "/bin/sh -c apt-get update" did not complete successfully: exit code: 100',
+      ),
+    ).toBe('');
+  });
+});
+
+describe('diagnose', () => {
+  test('any failing check makes the whole report not ok', () => {
+    const report = diagnose({
+      pkgDir: '/nonexistent',
+      probe: healthyProbe({ imageExists: () => false }),
+      lock: lockState(),
+    });
+    expect(report.ok).toBe(false);
+    expect(report.checks.find((c) => c.name === 'management-image')?.status).toBe('fail');
+  });
+
+  test('skipLock omits the lock check — run holds the lock itself', () => {
+    const report = diagnose({
+      pkgDir: '/nonexistent',
+      probe: healthyProbe(),
+      lock: lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }),
+      skipLock: true,
+    });
+    expect(report.checks.some((c) => c.name === 'run-lock')).toBe(false);
+    expect(report.ok).toBe(true);
+  });
+});