@celilo/e2e 0.19.3 → 0.20.0

package/src/doctor.test.ts

@@ -9,7 +9,11 @@
  * one.
  */
 
-import { describe, expect, test } from 'bun:test';
+import { afterAll, describe, expect, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import {
   BAKED_MANAGEMENT_IMAGES,
   type DockerfileBases,
@@ -17,22 +21,35 @@ import {
   EXPECTED_MANAGEMENT_CMD,
   checkBaseImages,
   checkDiskPressure,
+  checkHostVm,
+  checkImageFreshness,
+  checkLeakedStacks,
   checkManagementImage,
   checkRunLock,
   checkStaleContainers,
   diagnose,
   explainBuildFailure,
   parseBaseImages,
+  stackStartedAt,
 } from './doctor';
+import type { LeakedStack } from './doctor';
+import type { HostFacts, HostVmFacts } from './host-vm';
 import type { LockHolder, LockStatus } from './run-lock';
+import {
+  PUBLISHED_FINGERPRINT_PREFIX,
+  SOURCE_LABEL,
+  computeSourceFingerprint,
+} from './source-fingerprint';
 
 /** A probe where everything is healthy; each test breaks exactly one thing. */
 function healthyProbe(overrides: Partial<DoctorProbe> = {}): DoctorProbe {
   return {
     imageExists: () => true,
     imageCmd: () => [EXPECTED_MANAGEMENT_CMD],
+    imageLabel: () => null,
     staleContainers: () => [],
     reclaimableImageBytes: () => 0,
+    leakedStacks: () => [],
     ...overrides,
   };
 }
@@ -201,11 +218,18 @@ describe('stale containers and disk pressure', () => {
     expect(check.status).toBe('ok');
   });
 
-  test('large reclaimable space steers away from `docker image prune`', () => {
+  test('large reclaimable space offers the safe prune and names the destructive one', () => {
+    // The two forms are not interchangeable and the difference is the whole
+    // point: bare `docker image prune` removes UNTAGGED images (superseded
+    // management bakes, the bulk of this pile), while `-a` and `system prune`
+    // also remove the TAGGED base images the next build needs — which is the
+    // reflex that costs a full 27-image rebuild and then reads as a network
+    // failure. The remedy must say which is which.
     const check = checkDiskPressure(healthyProbe({ reclaimableImageBytes: () => 40 * 1024 ** 3 }));
     expect(check.status).toBe('warn');
-    expect(check.detail).toContain('docker image prune');
-    expect(check.remedy).toContain('name=celilo-e2e');
+    expect(check.remedy).toContain('docker image prune -f');
+    expect(check.remedy).toContain('NEVER `-a`');
+    expect(check.remedy).toContain('docker system prune');
   });
 
   test('disk pressure never blocks a run', () => {
@@ -219,6 +243,86 @@ describe('stale containers and disk pressure', () => {
   });
 });
 
+describe('leaked per-test stacks (ce-ywix: a crashed suite held a subnet for 2.5h)', () => {
+  // The mechanism: a Proxmox guest (celilo-e2e-lxc-<vmid>) carries no project
+  // name, survives every project-scoped sweep, and keeps the network attached —
+  // so every later run dies at compose up with "Pool overlaps". The check names
+  // the leak, its suite-of-origin timestamp, and the exact removal commands.
+  const leaked: LeakedStack[] = [
+    {
+      network: 'celilo-e2e-1788615872911_dmz',
+      containers: ['celilo-e2e-lxc-104'],
+    },
+  ];
+
+  test('a leaked stack with no live run warns and names the exact removal commands', () => {
+    const check = checkLeakedStacks(healthyProbe({ leakedStacks: () => leaked }), lockState());
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('celilo-e2e-1788615872911_dmz');
+    expect(check.detail).toContain('celilo-e2e-lxc-104');
+    expect(check.remedy).toContain('docker rm -f celilo-e2e-lxc-104');
+    expect(check.remedy).toContain('docker network rm celilo-e2e-1788615872911_dmz');
+  });
+
+  test('the suite-of-origin is attributable from the project timestamp', () => {
+    // The bead's attribution note: the project name embeds epoch-ms, so the
+    // leaker is identifiable after the fact without any other record.
+    const check = checkLeakedStacks(healthyProbe({ leakedStacks: () => leaked }), lockState());
+    expect(check.detail).toContain('suite started');
+    expect(check.detail).toMatch(/suite started \d+[smh]/);
+  });
+
+  test('a stack blocked by a NON-e2e container fails — the sweep cannot self-heal it', () => {
+    const check = checkLeakedStacks(
+      healthyProbe({
+        leakedStacks: () => [
+          { network: 'celilo-e2e-1788615872911_dmz', containers: ['my-dev-box'] },
+        ],
+      }),
+      lockState(),
+    );
+    expect(check.status).toBe('fail');
+    expect(check.detail).toContain('my-dev-box');
+    expect(check.remedy).toContain(
+      'docker network disconnect -f celilo-e2e-1788615872911_dmz my-dev-box',
+    );
+    expect(check.remedy).toContain('docker network rm celilo-e2e-1788615872911_dmz');
+    // Never instruct removing a container the sweep does not own.
+    expect(check.remedy).not.toContain('docker rm -f my-dev-box');
+  });
+
+  test("stacks belonging to a live holder are that run's, not debris", () => {
+    const check = checkLeakedStacks(
+      healthyProbe({ leakedStacks: () => leaked }),
+      lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }),
+    );
+    expect(check.status).toBe('ok');
+  });
+
+  test('the run\'s own preflight treats its held lock as "no legitimate holder"', () => {
+    // skipLock in the runner means WE hold the lock, so at preflight time any
+    // stack in sight is debris even though lockStatus() reads busy (our own
+    // holder record). Attributing it to "the holder" would hide the leak from
+    // the exact surface that exists to name it.
+    const check = checkLeakedStacks(
+      healthyProbe({ leakedStacks: () => leaked }),
+      lockState({ free: false, holder: holder(), heartbeatAgeMs: 0 }),
+      true,
+    );
+    expect(check.status).toBe('warn');
+  });
+
+  test('no stacks reads clean', () => {
+    expect(checkLeakedStacks(healthyProbe(), lockState()).status).toBe('ok');
+  });
+
+  test('the shared and interactive stacks are never mistaken for per-test debris', () => {
+    expect(stackStartedAt('celilo-e2e-shared_real-internet')).toBeNull();
+    expect(stackStartedAt('celilo-e2e-interactive_dmz')).toBeNull();
+    expect(stackStartedAt('celilo-e2e-1788615872911_dmz')).toBe(1788615872911);
+  });
+});
+
 describe('Dockerfile base parsing', () => {
   test("ignores references to the Dockerfile's own build stages", () => {
     const bases = parseBaseImages(
@@ -277,3 +381,154 @@ describe('diagnose', () => {
     expect(report.ok).toBe(true);
   });
 });
+
+// ─── The Docker host VM ──────────────────────────────────────────────
+//
+// Two settings, each measured on 2026-09-05 (see host-vm.ts). Warnings rather
+// than failures: a badly-shaped VM makes a run slow, not impossible, and this
+// check's job is to stop that being invisible.
+
+const M1_PRO: HostFacts = { cpus: 10, memoryGiB: 32 };
+
+function vm(overrides: Partial<HostVmFacts> = {}): HostVmFacts {
+  return {
+    profile: 'default',
+    cpus: 8,
+    memoryGiB: 12,
+    mountType: 'virtiofs',
+    vmType: 'vz',
+    ...overrides,
+  };
+}
+
+describe('checkHostVm', () => {
+  test('a VM inside the policy is ok and says what it found', () => {
+    const check = checkHostVm(vm(), M1_PRO);
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain('12 GiB');
+    expect(check.detail).toContain('virtiofs');
+  });
+
+  test('no VM at all is ok — docker runs natively on Linux', () => {
+    const check = checkHostVm(null, M1_PRO);
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain('no colima VM');
+  });
+
+  test('an oversized VM warns, because the host pays for the guest page cache', () => {
+    // The measured state: 24 of 32 GiB, host swapping 21.9 GiB.
+    const check = checkHostVm(vm({ memoryGiB: 24 }), M1_PRO);
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('24 GiB');
+    // Restarting fixes memory, so it must NOT demand the destructive path.
+    expect(check.remedy).toContain('cele2e host up');
+  });
+
+  test('an sshfs VM warns AND routes to reset, because colima cannot change it in place', () => {
+    const check = checkHostVm(vm({ mountType: 'sshfs' }), M1_PRO);
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('sshfs');
+    expect(check.remedy).toContain('cele2e host reset');
+  });
+
+  test('a qemu VM is not asked for virtiofs, which only Apple Virtualization has', () => {
+    const check = checkHostVm(vm({ vmType: 'qemu', mountType: 'sshfs' }), M1_PRO);
+    expect(check.status).toBe('ok');
+  });
+
+  test('more VM CPUs than the host has warns', () => {
+    const check = checkHostVm(vm({ cpus: 16 }), M1_PRO);
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('16 CPUs');
+  });
+});
+
+// ─── Image freshness ─────────────────────────────────────────────────
+//
+// The baked management image carries the celilo CLI every test exercises. A
+// stale one fails for reasons the working tree cannot explain, and says
+// nothing about it — so the fingerprint is compared, never assumed.
+
+/**
+ * A throwaway checkout, so these tests assert on a tree they control rather
+ * than on whatever the ambient worktree happens to hold. A test whose subject
+ * is "the repo I am running in" passes or fails for reasons unrelated to the
+ * code under test.
+ */
+function makeCheckout(): string {
+  const root = mkdtempSync(join(tmpdir(), 'fingerprint-'));
+  mkdirSync(join(root, 'apps', 'celilo', 'src'), { recursive: true });
+  writeFileSync(join(root, 'apps', 'celilo', 'package.json'), '{"name":"@celilo/cli"}\n');
+  writeFileSync(join(root, 'apps', 'celilo', 'src', 'index.ts'), 'export const a = 1;\n');
+  const git = (...args: string[]) =>
+    execFileSync('git', ['-C', root, ...args], { stdio: 'ignore' });
+  git('init', '-q');
+  git('config', 'user.email', 'test@celilo.invalid');
+  git('config', 'user.name', 'test');
+  git('add', '-A');
+  git('commit', '-qm', 'initial');
+  return root;
+}
+
+const REPO_ROOT = makeCheckout();
+afterAll(() => rmSync(REPO_ROOT, { recursive: true, force: true }));
+
+describe('checkImageFreshness', () => {
+  test('no checkout means nothing to be stale against', () => {
+    const check = checkImageFreshness(healthyProbe(), null);
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain('no celilo checkout');
+  });
+
+  test('an unstamped image warns, because its age cannot be established', () => {
+    // repoRoot is this checkout, so a fingerprint IS computed; the image has no label.
+    const check = checkImageFreshness(healthyProbe({ imageLabel: () => null }), REPO_ROOT);
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('no source stamp');
+    expect(check.remedy).toBe('cele2e build-infra');
+  });
+
+  test('a stamp from other source warns and names both fingerprints', () => {
+    const check = checkImageFreshness(
+      healthyProbe({ imageLabel: () => 'deadbeefdeadbeef' }),
+      REPO_ROOT,
+    );
+    expect(check.status).toBe('warn');
+    expect(check.detail).toContain('deadbeefdeadbeef');
+    expect(check.detail).toContain('NOT the code in this checkout');
+  });
+
+  test('a published bake is ok — it was never meant to match this tree', () => {
+    const check = checkImageFreshness(
+      healthyProbe({ imageLabel: () => `${PUBLISHED_FINGERPRINT_PREFIX}celilo 2.1.0` }),
+      REPO_ROOT,
+    );
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain('real npm');
+  });
+
+  test('a matching stamp is ok', () => {
+    const expected = computeSourceFingerprint(REPO_ROOT);
+    expect(expected).not.toBeNull();
+    const check = checkImageFreshness(
+      healthyProbe({ imageLabel: () => expected as string }),
+      REPO_ROOT,
+    );
+    expect(check.status).toBe('ok');
+    expect(check.detail).toContain('baked from this tree');
+  });
+
+  test('it reads the label the bake actually writes', () => {
+    let asked = '';
+    checkImageFreshness(
+      healthyProbe({
+        imageLabel: (_ref, label) => {
+          asked = label;
+          return null;
+        },
+      }),
+      REPO_ROOT,
+    );
+    expect(asked).toBe(SOURCE_LABEL);
+  });
+});

package/src/doctor.ts

@@ -17,6 +17,16 @@
 import { execFileSync } from 'node:child_process';
 import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
+import {
+  type HostFacts,
+  type HostVmFacts,
+  evaluateHostVm,
+  readHostFacts,
+  readHostVmFacts,
+  recommendedBudget,
+} from './host-vm';
+import { CONTAINER_PREFIX } from './proxmox-provisioner';
+import { findMonorepoRoot } from './repo-root';
 import {
   type LockHolder,
   type LockStatus,
@@ -26,14 +36,21 @@ import {
   isSuspect,
   lockStatus,
 } from './run-lock';
+import {
+  PUBLISHED_FINGERPRINT_PREFIX,
+  SOURCE_LABEL,
+  computeSourceFingerprint,
+} from './source-fingerprint';
 
 /**
  * Images the per-test compose references by `image:` alone — it has no `build:`
  * for them, so docker CANNOT produce them on demand. `cele2e build-infra` bakes
  * both (install.sh, then `docker commit`). Missing → the run is doomed.
  */
+export const MANAGEMENT_LATEST = 'celilo-e2e/management:latest';
+
 export const BAKED_MANAGEMENT_IMAGES = [
-  'celilo-e2e/management:latest',
+  MANAGEMENT_LATEST,
   'celilo-e2e/management:vanilla',
 ] as const;
 
@@ -68,16 +85,32 @@ export interface DoctorReport {
   ok: boolean;
 }
 
+/**
+ * A leaked per-test compose stack: one timestamped-project network plus whatever
+ * is still attached to it. The project name embeds the epoch-ms the suite
+ * started, which is what makes the leaker attributable after the fact.
+ */
+export interface LeakedStack {
+  /** e.g. `celilo-e2e-1788615872911_dmz` */
+  network: string;
+  /** Attached container names, e.g. `celilo-e2e-lxc-104`. */
+  containers: string[];
+}
+
 /** Every Docker fact doctor needs, injectable so the checks are testable. */
 export interface DoctorProbe {
   /** Is this image present in the local store? An untagged ref means `:latest`. */
   imageExists(ref: string): boolean;
   /** The image's configured Cmd, or null when the image is absent. */
   imageCmd(ref: string): string[] | null;
+  /** One label off the image, or null when the image or the label is absent. */
+  imageLabel(ref: string, label: string): string | null;
   /** Names of leftover `celilo-e2e-*` containers. */
   staleContainers(): string[];
   /** Bytes docker reports as reclaimable image space. */
   reclaimableImageBytes(): number;
+  /** Per-test compose networks still up, with their attached containers. */
+  leakedStacks(): LeakedStack[];
 }
 
 // ─── Dockerfile parsing ──────────────────────────────────────────────
@@ -233,6 +266,89 @@ export function checkBaseImages(probe: DoctorProbe, dockerfiles: DockerfileBases
   };
 }
 
+/**
+ * Epoch-ms a per-test stack started, parsed from its network's project name —
+ * the attribution `docker network ls` carries for free. Null when the name is
+ * not a per-test project (the shared and interactive projects hold no digits).
+ */
+export function stackStartedAt(network: string): number | null {
+  const ms = network.match(/^celilo-e2e-(\d{13})_/)?.[1];
+  return ms ? Number(ms) : null;
+}
+
+/**
+ * Networks from a crashed run that hold the sim's subnets.
+ *
+ * An active run holds the machine-global run-lock, so outside one there is no
+ * legitimate holder of a `celilo-e2e-<ts>_*` network — it is always teardown
+ * debris. Left in place it blocks EVERY later run with
+ * `invalid pool request: Pool overlaps` at compose up (ce-ywix), a failure that
+ * arrived only after minutes of image builds. The start-of-run sweep removes
+ * these networks and their e2e-attached containers, so a cleanable leak is a
+ * warning; a network blocked by a NON-e2e container is a failure, because the
+ * sweep deliberately never touches foreign containers and the run cannot
+ * succeed.
+ */
+export function checkLeakedStacks(
+  probe: DoctorProbe,
+  lock: LockStatus,
+  callerHoldsLock = false,
+): DoctorCheck {
+  const stacks = probe.leakedStacks();
+  if (stacks.length === 0) {
+    return { name: 'leaked-stacks', status: 'ok', detail: 'no leaked per-test stack networks' };
+  }
+  // A live holder owns whatever per-test stack exists right now. `run` passes
+  // callerHoldsLock instead: it already owns the lock, so at preflight time
+  // every stack it can see is by definition somebody's debris, never its own.
+  if (!callerHoldsLock && !lock.free && !lock.ownKept) {
+    return {
+      name: 'leaked-stacks',
+      status: 'ok',
+      detail: `${stacks.length} per-test network(s) belonging to the current lock holder`,
+    };
+  }
+
+  const describe = (s: LeakedStack): string => {
+    const startedAt = stackStartedAt(s.network);
+    const age = startedAt ? ` (suite started ${formatAge(Date.now() - startedAt)} ago)` : '';
+    const attached = s.containers.length > 0 ? `, attached: ${s.containers.join(', ')}` : '';
+    return `${s.network}${age}${attached}`;
+  };
+
+  const foreign = stacks.flatMap((s) =>
+    s.containers.filter((c) => !c.startsWith(CONTAINER_PREFIX)),
+  );
+  if (foreign.length > 0) {
+    const blocked = stacks.filter((s) => s.containers.some((c) => !c.startsWith(CONTAINER_PREFIX)));
+    return {
+      name: 'leaked-stacks',
+      status: 'fail',
+      detail: `leaked per-test network(s) with a NON-e2e container attached — the cleanup sweep never touches foreign containers, so this stack cannot self-heal: ${blocked.map(describe).join('; ')}`,
+      remedy: blocked
+        .flatMap((s) =>
+          s.containers
+            .filter((c) => !c.startsWith(CONTAINER_PREFIX))
+            .map((c) => `docker network disconnect -f ${s.network} ${c}`),
+        )
+        .concat(blocked.map((s) => `docker network rm ${s.network}`))
+        .join('\n           '),
+    };
+  }
+
+  return {
+    name: 'leaked-stacks',
+    status: 'warn',
+    detail: `leaked per-test stack(s) holding the sim's subnets — always teardown debris, a later run's start-of-run cleanup removes them: ${stacks.slice(0, 3).map(describe).join('; ')}${stacks.length > 3 ? '; …' : ''}`,
+    remedy: [
+      ...stacks.flatMap((s) =>
+        s.containers.length > 0 ? [`docker rm -f ${s.containers.join(' ')}`] : [],
+      ),
+      ...stacks.map((s) => `docker network rm ${s.network}`),
+    ].join('\n           '),
+  };
+}
+
 /** Leftover containers from a crashed run, which start-of-run cleanup will wipe. */
 export function checkStaleContainers(probe: DoctorProbe, lock: LockStatus): DoctorCheck {
   const names = probe.staleContainers();
@@ -259,6 +375,93 @@ export function checkStaleContainers(probe: DoctorProbe, lock: LockStatus): Doct
   };
 }
 
+/**
+ * The Docker host's virtual machine, on the two settings that dominate run
+ * time. Never a failure: a run on a badly-shaped VM is slow, not doomed, and
+ * refusing to run would be a worse trade than saying so. Silent on Linux and
+ * anywhere without a colima VM, where there is nothing to shape.
+ *
+ * See `host-vm.ts` for the measurements behind each rule.
+ */
+export function checkHostVm(facts: HostVmFacts | null, host: HostFacts): DoctorCheck {
+  if (!facts) {
+    return {
+      name: 'docker-host',
+      status: 'ok',
+      detail: 'no colima VM — docker runs natively, no host-VM policy applies',
+    };
+  }
+  const budget = recommendedBudget(host, facts.vmType);
+  const { problems, needsRecreate } = evaluateHostVm(facts, host, budget);
+  if (problems.length === 0) {
+    return {
+      name: 'docker-host',
+      status: 'ok',
+      detail: `colima/${facts.profile}: ${facts.cpus} CPU, ${facts.memoryGiB} GiB, ${facts.mountType} mounts`,
+    };
+  }
+  return {
+    name: 'docker-host',
+    status: 'warn',
+    detail: problems.join(' '),
+    remedy: needsRecreate
+      ? 'cele2e host reset   (colima discards a mount-type change on an existing VM, so this DESTROYS the VM and its image cache and rebuilds — budget one `cele2e build-infra`)'
+      : 'cele2e host up   (restarts the VM with the recommended budget; images survive)',
+  };
+}
+
+/**
+ * Was the baked management image built from the source in this working tree?
+ *
+ * The image carries a real installed `celilo`, and that CLI is what every test
+ * exercises. A stale one produces a run that passes or fails for reasons the
+ * working tree cannot explain, with nothing in the output to say so. Comparing
+ * a fingerprint stamped at bake time against one computed now turns that into a
+ * sentence.
+ *
+ * A warning, not a failure — deliberately testing an older image is a real
+ * thing to do, and this check's job is to make sure it is deliberate.
+ */
+export function checkImageFreshness(probe: DoctorProbe, repoRoot: string | null): DoctorCheck {
+  const expected = computeSourceFingerprint(repoRoot ?? undefined);
+  if (!expected) {
+    return {
+      name: 'image-freshness',
+      status: 'ok',
+      detail: 'no celilo checkout to compare the baked image against',
+    };
+  }
+  const stamped = probe.imageLabel(MANAGEMENT_LATEST, SOURCE_LABEL);
+  if (stamped === null) {
+    return {
+      name: 'image-freshness',
+      status: 'warn',
+      detail: `${MANAGEMENT_LATEST} carries no source stamp, so it predates this check and its age cannot be established`,
+      remedy: 'cele2e build-infra',
+    };
+  }
+  if (stamped.startsWith(PUBLISHED_FINGERPRINT_PREFIX)) {
+    return {
+      name: 'image-freshness',
+      status: 'ok',
+      detail: `${MANAGEMENT_LATEST} was baked from ${stamped} (real npm), not from this tree — nothing to be stale against`,
+    };
+  }
+  if (stamped !== expected) {
+    return {
+      name: 'image-freshness',
+      status: 'warn',
+      detail: `${MANAGEMENT_LATEST} was baked from source ${stamped}; this tree is ${expected}. The celilo CLI under test is NOT the code in this checkout`,
+      remedy: 'cele2e build-infra',
+    };
+  }
+  return {
+    name: 'image-freshness',
+    status: 'ok',
+    detail: `${MANAGEMENT_LATEST} was baked from this tree (${stamped})`,
+  };
+}
+
 /**
  * Reclaimable image space. A warning, never a failure — the point is to steer
  * the reflex: under disk pressure people reach for `docker image prune`, which
@@ -274,9 +477,9 @@ export function checkDiskPressure(probe: DoctorProbe): DoctorCheck {
   return {
     name: 'disk',
     status: 'warn',
-    detail: `${gib} GiB reclaimable image space — do NOT run \`docker image prune\`; it deletes the base images the next build needs and costs a full rebuild`,
+    detail: `${gib} GiB reclaimable image space — most of it is superseded management images, which \`cele2e build-infra\` now removes as it goes`,
     remedy:
-      'docker ps -aq --filter name=celilo-e2e | xargs -r docker rm -f && docker volume prune -f',
+      'docker image prune -f   (UNTAGGED images only — safe. NEVER `-a` or `docker system prune`, which delete the TAGGED base images the next build needs and cost a full 27-image rebuild)',
   };
 }
 
@@ -347,6 +550,21 @@ export function createDockerProbe(): DoctorProbe {
         return null;
       }
     },
+    imageLabel(ref, label) {
+      try {
+        const out = docker([
+          'image',
+          'inspect',
+          ref,
+          '--format',
+          `{{index .Config.Labels "${label}"}}`,
+        ]);
+        // Go's text/template renders a missing map key as this literal.
+        return out === '' || out === '<no value>' ? null : out;
+      } catch {
+        return null;
+      }
+    },
     staleContainers() {
       try {
         return docker(['ps', '-a', '--filter', 'name=celilo-e2e', '--format', '{{.Names}}'])
@@ -357,6 +575,38 @@ export function createDockerProbe(): DoctorProbe {
         return [];
       }
     },
+    leakedStacks() {
+      // Per-test compose networks only: the project carries the epoch-ms the
+      // suite started, so a digits-then-underscore name cannot match the
+      // shared (`celilo-e2e-shared_*`) or interactive (`celilo-e2e-interactive`)
+      // stacks, both of which can be legitimately up outside a run.
+      const perTest = (n: string): boolean => /^celilo-e2e-\d{13}_/.test(n);
+      try {
+        const nets = docker(['network', 'ls', '--format', '{{.Name}}'])
+          .split('\n')
+          .map((l) => l.trim())
+          .filter(perTest);
+        const stacks: LeakedStack[] = [];
+        for (const net of nets) {
+          try {
+            const containers = JSON.parse(
+              docker(['network', 'inspect', net, '--format', '{{json .Containers}}']),
+            ) as Record<string, { Name?: string }>;
+            stacks.push({
+              network: net,
+              containers: Object.values(containers)
+                .map((c) => c.Name ?? '')
+                .filter(Boolean),
+            });
+          } catch {
+            stacks.push({ network: net, containers: [] });
+          }
+        }
+        return stacks;
+      } catch {
+        return [];
+      }
+    },
     reclaimableImageBytes() {
       try {
         for (const line of docker(['system', 'df', '--format', '{{json .}}']).split('\n')) {
@@ -384,6 +634,16 @@ export interface DiagnoseOptions {
   lock?: LockStatus;
   /** Skip the run-lock check — `run` acquires the lock itself and reports contention its own way. */
   skipLock?: boolean;
+  /**
+   * The celilo checkout to fingerprint against. Defaults to a walk-up from
+   * `pkgDir`, so a caller cannot silently disable the freshness check by
+   * forgetting to pass it — the failure mode this repo calls "a check that
+   * cannot reach the thing it is checking".
+   */
+  repoRoot?: string | null;
+  /** The running VM and the host it sits on. Injected so both are testable. */
+  hostVm?: HostVmFacts | null;
+  host?: HostFacts;
 }
 
 export function diagnose(options: DiagnoseOptions): DoctorReport {
@@ -394,6 +654,19 @@ export function diagnose(options: DiagnoseOptions): DoctorReport {
   checks.push(checkManagementImage(probe));
   checks.push(checkBaseImages(probe, readDockerfileBases(options.pkgDir)));
   checks.push(checkStaleContainers(probe, lock));
+  checks.push(
+    checkImageFreshness(
+      probe,
+      options.repoRoot !== undefined ? options.repoRoot : findMonorepoRoot(options.pkgDir),
+    ),
+  );
+  checks.push(
+    checkHostVm(
+      options.hostVm !== undefined ? options.hostVm : readHostVmFacts(),
+      options.host ?? readHostFacts(),
+    ),
+  );
+  checks.push(checkLeakedStacks(probe, lock, options.skipLock === true));
   checks.push(checkDiskPressure(probe));
   return { checks, ok: !checks.some((c) => c.status === 'fail') };
 }