@celilo/e2e 0.19.3 → 0.20.1

package/src/container-manager.cleanup.test.ts

@@ -0,0 +1,284 @@
+/**
+ * Lane A2 of openspec/changes/e2e-suite-recovery (ce-yuzi): the best-effort
+ * cleanup sweeps must not report success they did not achieve.
+ *
+ * Every sweep step that fails is recorded as a CleanupFailure and logged as a
+ * `[cleanup:failed]` line; the sweep itself still never throws (the exit
+ * handler and the pre-suite recovery sweep must run to completion even when
+ * docker is gone). Reach is measured, not reasoned about: a fake DockerCli
+ * emulates docker's own filter semantics over a planted world, the REAL
+ * unmodified sweep runs against it, and the rm commands it issues are compared
+ * against the set of resources the sweep was supposed to reach.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  cleanupProgress,
+  forceRemoveProject,
+  sweepStaleTestResources,
+  withDockerCli,
+} from './container-manager';
+
+const PROJECT = 'celilo-e2e-1788449289917';
+
+/** A planted docker world keyed by resource kind. Guest carries its label;
+ * `attached` names the containers docker reports on each network. */
+interface DockerWorld {
+  containers: Record<string, string>; // name -> id
+  guests: Record<string, { id: string; project?: string }>; // name -> guest
+  networks: string[];
+  attached: Record<string, string[]>; // network -> attached container names
+  volumes: string[];
+}
+
+/** A fake DockerCli.exec that answers docker listing commands from `world`
+ * using docker's own filter semantics (name filter = substring match, label
+ * filter = exact), records every `rm` it is asked to run, and throws on
+ * `failOn` commands. Anything it cannot answer throws — a fixture that
+ * silently answers `''` would understate reach exactly like the real sweep
+ * skipping a directory does. */
+function fakeDocker(world: DockerWorld, failOn?: (cmd: string) => boolean) {
+  const rms: string[] = [];
+  const exec = (cmd: string): string => {
+    if (failOn?.(cmd)) throw new Error(`docker: ${cmd.split(' ').slice(0, 3).join(' ')} failed`);
+    if (
+      cmd.startsWith('docker rm -f ') ||
+      cmd.startsWith('docker network rm ') ||
+      cmd.startsWith('docker volume rm ')
+    ) {
+      rms.push(cmd);
+      return '';
+    }
+    const nameFilter = (prefix: string): string | undefined => {
+      const m = cmd.slice(prefix.length).match(/--filter "?name=([^\s"']+)/);
+      return m && cmd.startsWith(prefix) ? m[1] : undefined;
+    };
+    // Container listing (by name and by guest label)
+    if (cmd.startsWith('docker ps -aq')) {
+      const label = cmd.match(/--filter label=(\S+?)(?:=(\S+))?$/);
+      if (label) {
+        return Object.entries(world.guests)
+          .filter(([, g]) => label[2] === undefined || g.project === label[2])
+          .map(([, g]) => g.id)
+          .join('\n');
+      }
+      const nf = nameFilter('docker ps');
+      if (nf === undefined) throw new Error(`fakeDocker: unanswerable: ${cmd}`);
+      return Object.entries(world.containers)
+        .filter(([name]) => name.includes(nf))
+        .map(([, id]) => id)
+        .join('\n');
+    }
+    if (cmd.includes('docker network ls')) {
+      const nf = cmd.match(/--filter name=([^\s"']+)/)?.[1];
+      return world.networks.filter((n) => (nf === undefined ? true : n.includes(nf))).join('\n');
+    }
+    if (cmd.startsWith('docker network inspect ')) {
+      const net = cmd.match(/docker network inspect (\S+)/)?.[1] ?? '';
+      const attached = (world.attached[net] ?? []).map((name, i) => [
+        `deadbeef${i}`,
+        { Name: name },
+      ]);
+      return JSON.stringify(Object.fromEntries(attached));
+    }
+    if (cmd.startsWith('docker volume ls -q')) {
+      const nf = cmd.match(/--filter name=([^\s"']+)/)?.[1] ?? '';
+      return world.volumes.filter((v) => v.includes(nf)).join('\n');
+    }
+    throw new Error(`fakeDocker: unanswerable: ${cmd}`);
+  };
+  return { exec, rms };
+}
+
+function plantedWorld(): DockerWorld {
+  return {
+    containers: {
+      [`${PROJECT}_fw-main_1`]: 'cid-fw',
+      [`${PROJECT}_caddy_1`]: 'cid-caddy',
+      // A live non-e2e container: no filter may ever reach it.
+      'nginx-live': 'cid-nginx',
+    },
+    guests: {
+      'celilo-e2e-lxc-7001': { id: 'cid-guest', project: PROJECT },
+    },
+    networks: [`${PROJECT}_zone-a`, `${PROJECT}_zone-b`],
+    attached: {
+      [`${PROJECT}_zone-a`]: ['celilo-e2e-lxc-7001', 'hand-attached'],
+      [`${PROJECT}_zone-b`]: [],
+    },
+    volumes: [`${PROJECT}_ssh-keys`],
+  };
+}
+
+describe('forceRemoveProject reach (measured, not reasoned)', () => {
+  test('removes every planted container, guest, network and volume of the project', () => {
+    const world = plantedWorld();
+    const { exec, rms } = fakeDocker(world);
+    withDockerCli({ exec, spawn: undefined as never }, () => {
+      forceRemoveProject(PROJECT);
+    });
+    const removed = rms.join('\n');
+    // Containers and the label-listed guest (celilo#1247).
+    expect(removed).toContain('cid-fw');
+    expect(removed).toContain('cid-caddy');
+    expect(removed).toContain('cid-guest');
+    // Both networks.
+    expect(removed).toContain(`${PROJECT}_zone-a`);
+    expect(removed).toContain(`${PROJECT}_zone-b`);
+    // The volume.
+    expect(removed).toContain(`${PROJECT}_ssh-keys`);
+    // Containers attached to a project network that carry the provisioner's
+    // CONTAINER_PREFIX containment contract (the guest does) get their own rm.
+    expect(rms.some((c) => c.startsWith('docker rm -f') && c.includes('celilo-e2e-lxc-7001'))).toBe(
+      true,
+    );
+  });
+
+  test('never reaches a decoy: shared infra, hand-attached containers, non-e2e', () => {
+    const world = plantedWorld();
+    const { exec, rms } = fakeDocker(world);
+    withDockerCli({ exec, spawn: undefined as never }, () => {
+      forceRemoveProject(PROJECT);
+    });
+    const removed = rms.join('\n');
+    expect(removed).not.toContain('nginx-live');
+    expect(removed).not.toContain('hand-attached');
+    expect(removed).not.toContain('celilo-e2e-shared');
+  });
+});
+
+describe('forceRemoveProject failures surface (ce-yuzi)', () => {
+  test('a docker rm that throws is reported, not swallowed', () => {
+    const world = plantedWorld();
+    const { exec } = fakeDocker(world, (cmd) => cmd.startsWith('docker rm -f'));
+    const failures = withDockerCli({ exec, spawn: undefined as never }, () =>
+      forceRemoveProject(PROJECT),
+    );
+    expect(failures.length).toBeGreaterThan(0);
+    expect(failures.some((f) => f.step.includes('docker rm -f'))).toBe(true);
+    expect(failures.some((f) => f.detail.includes('failed'))).toBe(true);
+  });
+
+  test('a network rm that throws is reported, and later steps still ran', () => {
+    const world = plantedWorld();
+    const { exec, rms } = fakeDocker(world, (cmd) => cmd.startsWith('docker network rm'));
+    const failures = withDockerCli({ exec, spawn: undefined as never }, () =>
+      forceRemoveProject(PROJECT),
+    );
+    expect(failures.some((f) => f.step.includes('docker network rm'))).toBe(true);
+    // Volumes come after networks: a failed network rm must not stop them.
+    expect(rms.some((c) => c.startsWith('docker volume rm'))).toBe(true);
+  });
+
+  test('the sweep never throws, even when every docker command fails', () => {
+    const { exec } = fakeDocker(plantedWorld(), () => true);
+    const failures = withDockerCli({ exec, spawn: undefined as never }, () =>
+      forceRemoveProject(PROJECT),
+    );
+    expect(failures.length).toBeGreaterThan(0);
+  });
+
+  test('a LISTING that fails is recorded, not read as an empty result', () => {
+    // An empty answer and a broken docker look identical to a sweep that
+    // filters; the list failure must be counted or the sweep reports success
+    // over resources it never saw.
+    const { exec } = fakeDocker(plantedWorld(), (cmd) =>
+      cmd.startsWith('docker ps -aq --filter name='),
+    );
+    const failures = withDockerCli({ exec, spawn: undefined as never }, () =>
+      forceRemoveProject(PROJECT),
+    );
+    expect(failures.some((f) => f.step.startsWith('docker ps name='))).toBe(true);
+  });
+});
+
+describe('sweepStaleTestResources reach (measured, not reasoned)', () => {
+  function staleWorld(): DockerWorld {
+    return {
+      containers: {
+        [`${PROJECT}_fw-main_1`]: 'cid-fw',
+        'celilo-e2e-shared-namecheap-dns-1': 'cid-shared-dns',
+        'nginx-live': 'cid-nginx',
+      },
+      guests: {
+        'celilo-e2e-lxc-7001': { id: 'cid-guest' },
+      },
+      networks: [`${PROJECT}_zone-a`, 'celilo-e2e-shared_namecheap'],
+      attached: {},
+      volumes: [`${PROJECT}_ssh-keys`, 'celilo-e2e-shared_dns'],
+    };
+  }
+
+  test('reaches per-test containers, guests of any project, project networks and volumes', () => {
+    const { exec, rms } = fakeDocker(staleWorld());
+    const failures = withDockerCli({ exec, spawn: undefined as never }, () =>
+      sweepStaleTestResources(),
+    );
+    expect(failures).toEqual([]);
+    const removed = rms.join('\n');
+    expect(removed).toContain('cid-fw');
+    expect(removed).toContain('cid-guest');
+    expect(removed).toContain(`${PROJECT}_zone-a`);
+    expect(removed).toContain(`${PROJECT}_ssh-keys`);
+  });
+
+  test('never reaches shared infra or non-e2e resources', () => {
+    const { exec, rms } = fakeDocker(staleWorld());
+    withDockerCli({ exec, spawn: undefined as never }, () => sweepStaleTestResources());
+    const removed = rms.join('\n');
+    expect(removed).not.toContain('cid-shared-dns');
+    expect(removed).not.toContain('nginx-live');
+    expect(removed).not.toContain('celilo-e2e-shared');
+  });
+
+  test('sweep failures are returned, not swallowed', () => {
+    const { exec } = fakeDocker(staleWorld(), (cmd) => cmd.startsWith('docker rm -f'));
+    const failures = withDockerCli({ exec, spawn: undefined as never }, () =>
+      sweepStaleTestResources(),
+    );
+    expect(failures.length).toBeGreaterThan(0);
+  });
+});
+
+describe('cleanupProgress is honest by construction', () => {
+  test('says complete only on zero failures', () => {
+    expect(cleanupProgress([])).toBe('cleanup complete');
+  });
+
+  test('says incomplete and names the count when steps failed', () => {
+    const line = cleanupProgress([
+      { step: 'docker rm -f x', detail: 'boom' },
+      { step: 'docker network rm y', detail: 'boom' },
+    ]);
+    expect(line).not.toContain('cleanup complete');
+    expect(line).toContain('2');
+    expect(line).toContain('[cleanup:failed]');
+  });
+});
+
+describe('the honest progress line is actually wired into startNetwork', () => {
+  // Same source-assertion style as exit-cleanup.test.ts: behavior tests above
+  // prove the helpers, this proves startNetwork really uses them.
+  const SRC = readFileSync(
+    join(dirname(fileURLToPath(import.meta.url)), 'container-manager.ts'),
+    'utf-8',
+  );
+
+  test('startNetwork prints the sweep result after sweeping, via cleanupProgress', () => {
+    const body = SRC.slice(SRC.indexOf('export async function startNetwork'));
+    const sweepCall = body.indexOf('sweepStaleTestResources()');
+    const progressCall = body.indexOf('cleanupProgress(cleanupFailures)');
+    expect(sweepCall).toBeGreaterThan(-1);
+    expect(progressCall).toBeGreaterThan(sweepCall);
+  });
+
+  test('no path prints "cleanup complete" except through cleanupProgress', () => {
+    // The only literal left in the file is inside cleanupProgress itself.
+    const idx = SRC.indexOf("'cleanup complete'");
+    expect(idx).toBeGreaterThan(-1);
+    expect(SRC.lastIndexOf('export function cleanupProgress', idx)).toBeGreaterThan(-1);
+  });
+});

package/src/container-manager.runner.test.ts

@@ -0,0 +1,351 @@
+import { describe, expect, test } from 'bun:test';
+import type { ExecSyncOptions } from 'node:child_process';
+/**
+ * The container manager's docker access, driven through the injected
+ * DockerCli — no daemon involved.
+ *
+ * This is the seam lane A1 (ce-h4no) exists to create. Everything before it
+ * was untestable: the cleanup cascade, the exec wrappers and the image
+ * accounting all reached for `execSync`/`spawn` directly, so nothing here
+ * could run under a unit test. The functions that already carry their own
+ * injection (`removeAlienInterface`, proxmox-provisioner.test.ts) or are
+ * pure (`projectTeardownCommands`, exit-cleanup.test.ts) were already covered.
+ */
+import {
+  type DockerCli,
+  dockerExec,
+  forceRemoveProject,
+  missingImages,
+  plainDockerExec,
+  projectTeardownCommands,
+  scrubDnsZones,
+  withDockerCli,
+} from './container-manager';
+import { SHARED_PROJECT_NAME } from './docker-compose-generator';
+import { SIMULATOR_IPS } from './simulator-ips';
+
+/**
+ * A recording fake: `exec` answers from a script of substring → output,
+ * `spawn` throws (none of the code under test here may spawn). Commands are
+ * recorded in order so tests can assert the sweep's ordering.
+ */
+function fakeDockerCli(script: Record<string, string>): { cli: DockerCli; commands: string[] } {
+  const commands: string[] = [];
+  return {
+    commands,
+    cli: {
+      exec(command: string, _opts?: ExecSyncOptions): string {
+        commands.push(command);
+        for (const [needle, output] of Object.entries(script)) {
+          if (command.includes(needle)) return output;
+        }
+        throw new Error(`fake docker has no answer for: ${command}`);
+      },
+      spawn(args: string[]): never {
+        throw new Error(`fake docker must not spawn: docker ${args.join(' ')}`);
+      },
+    },
+  };
+}
+
+const TEST_PROJECT = 'celilo-e2e-1726123456789';
+
+describe('forceRemoveProject', () => {
+  test('removes containers, guests, networks and volumes, in that order', () => {
+    const { cli, commands } = fakeDockerCli({
+      [`--filter name=${TEST_PROJECT}`]: 'c1\nc2\n',
+      'celilo-e2e.project': 'g1\n',
+      'network ls': 'celilo-e2e-1726123456789_app\n',
+      'volume ls': 'v1\n',
+    });
+
+    withDockerCli(cli, () => forceRemoveProject(TEST_PROJECT));
+
+    // Containers and label-matched guests go in one rm -f, before any network
+    // or volume removal: a guest holds its zone network's endpoint, and a
+    // volume cannot be removed while a container still attaches it.
+    const containerRm = commands.findIndex((c) => c.includes('docker rm -f c1 c2 g1'));
+    const networkRm = commands.findIndex((c) => c.includes('docker network rm'));
+    const volumeRm = commands.findIndex((c) => c.includes('docker volume rm'));
+    expect(containerRm).toBeGreaterThanOrEqual(0);
+    expect(networkRm).toBeGreaterThan(containerRm);
+    expect(volumeRm).toBeGreaterThan(networkRm);
+    // The sweep keys off the per-test project only. Shared infra (DNS, ACME,
+    // registry simulators) must never appear in a removal command.
+    for (const command of commands.filter((c) => c.includes('rm'))) {
+      expect(command).not.toContain(SHARED_PROJECT_NAME);
+    }
+  });
+
+  test('removes by label as well as by name, using the shared teardown commands', () => {
+    const { cli, commands } = fakeDockerCli({
+      [`--filter name=${TEST_PROJECT}`]: '',
+      'celilo-e2e.project': 'g9\n',
+    });
+    withDockerCli(cli, () => forceRemoveProject(TEST_PROJECT));
+    // The label filter is what catches a sim-created guest: its name
+    // (celilo-e2e-lxc-<vmid>) carries no timestamp, so the name filter
+    // cannot match it, and an unremoved guest pins its zone network.
+    expect(commands).toContain(projectTeardownCommands(TEST_PROJECT).listGuests);
+    expect(commands.some((c) => c.includes('docker rm -f g9'))).toBe(true);
+  });
+
+  test('survives docker being down entirely — no throw, no removal attempted', () => {
+    const { cli, commands } = fakeDockerCli({});
+    // Every listing throws (no script entry matches). The sweep is the exit
+    // handler's backstop: it must complete, not crash the handler.
+    expect(() => withDockerCli(cli, () => forceRemoveProject(TEST_PROJECT))).not.toThrow();
+    expect(commands.filter((c) => c.includes('rm -f') || c.includes(' rm '))).toEqual([]);
+  });
+
+  test('keeps sweeping when one removal fails', () => {
+    const ran: string[] = [];
+    const cli: DockerCli = {
+      exec(command: string) {
+        ran.push(command);
+        // Substring checks run specific-first: listNetworks contains BOTH
+        // 'network ls' and `--filter name=<project>`, so the generic name
+        // check must not win.
+        if (command.includes('docker rm -f')) throw new Error('device or resource busy');
+        if (command.includes('network ls')) return 'n1\n';
+        if (command.includes('volume ls')) return 'v1\n';
+        if (command.includes(`--filter name=${TEST_PROJECT}`)) return 'c1\n';
+        throw new Error(`no answer for: ${command}`);
+      },
+      spawn() {
+        throw new Error('must not spawn');
+      },
+    };
+    expect(() => withDockerCli(cli, () => forceRemoveProject(TEST_PROJECT))).not.toThrow();
+    // The network and volume removals still ran after the container rm failed.
+    expect(ran.some((c) => c.includes('docker network rm n1'))).toBe(true);
+    expect(ran.some((c) => c.includes('docker volume rm v1'))).toBe(true);
+  });
+});
+
+describe('dockerExec', () => {
+  test('routes a project container through the per-test compose project', () => {
+    const { cli, commands } = fakeDockerCli({ 'bash -c': 'pong' });
+    const result = withDockerCli(cli, () =>
+      dockerExec(TEST_PROJECT, '/tmp/compose', 'management', 'celilo status', 5_000),
+    );
+    expect(result).toEqual({ stdout: 'pong', stderr: '', exitCode: 0 });
+    expect(commands[0]).toContain('-f docker-compose.test.yml');
+    expect(commands[0]).toContain(`-p ${TEST_PROJECT}`);
+    expect(commands[0]).toContain('exec -T management bash -c');
+    expect(commands[0]).not.toContain(SHARED_PROJECT_NAME);
+  });
+
+  test('routes a shared container through the shared compose project', () => {
+    const { cli, commands } = fakeDockerCli({ 'bash -c': 'pong' });
+    withDockerCli(cli, () => dockerExec(TEST_PROJECT, '/tmp/compose', 'namecheap-dns', 'true'));
+    expect(commands[0]).toContain('-f docker-compose.shared.yml');
+    expect(commands[0]).toContain(`-p ${SHARED_PROJECT_NAME}`);
+  });
+
+  test('maps a timeout to exitCode 124 and an actionable stderr', () => {
+    const timeoutCli: DockerCli = {
+      exec() {
+        const err = new Error('spawn sync ETIMEDOUT') as Error & { code?: string };
+        err.code = 'ETIMEDOUT';
+        throw err;
+      },
+      spawn() {
+        throw new Error('must not spawn');
+      },
+    };
+    const result = withDockerCli(timeoutCli, () =>
+      dockerExec(TEST_PROJECT, '/tmp/compose', 'management', 'celilo init', 8_000),
+    );
+    expect(result.exitCode).toBe(124);
+    expect(result.stderr).toContain('timed out after 8s');
+    expect(result.stderr).toContain('celilo init');
+  });
+
+  // Real shape measured 2026-09-06 (ce-013r / celilo#1293): execSync fires its
+  // timer and SIGTERMs the `docker compose exec` client; compose traps the
+  // signal, cleans up, and EXITS 130 by its own convention. The error then
+  // carries status: 130, and `e.status ?? 124` adopted it — the harness
+  // reported its own timeout as a mystery "exit 130" from the exec'd command.
+  // The client's death code is not the command's verdict: a timed-out exec is
+  // always 124, whatever status the dying client left behind.
+  test('a timeout whose compose client exits 130 still reports 124', () => {
+    const timeoutCli: DockerCli = {
+      exec() {
+        const err = new Error('spawn sync ETIMEDOUT') as Error & {
+          code?: string;
+          killed?: boolean;
+          status?: number;
+        };
+        err.code = 'ETIMEDOUT';
+        err.killed = true;
+        err.status = 130;
+        throw err;
+      },
+      spawn() {
+        throw new Error('must not spawn');
+      },
+    };
+    const result = withDockerCli(timeoutCli, () =>
+      dockerExec(
+        TEST_PROJECT,
+        '/tmp/compose',
+        'celilo-mgr-2',
+        'celilo module import celilo-mgmt',
+        120_000,
+      ),
+    );
+    expect(result.exitCode).toBe(124);
+    expect(result.stderr).toContain('timed out after 120s');
+  });
+
+  test('plainDockerExec maps a timeout to exitCode 124, not the client status', () => {
+    const timeoutCli: DockerCli = {
+      exec() {
+        const err = new Error('spawn sync ETIMEDOUT') as Error & {
+          code?: string;
+          killed?: boolean;
+          status?: number;
+        };
+        err.code = 'ETIMEDOUT';
+        err.killed = true;
+        err.status = 130;
+        throw err;
+      },
+      spawn() {
+        throw new Error('must not spawn');
+      },
+    };
+    const result = withDockerCli(timeoutCli, () =>
+      plainDockerExec('provisioned-guest', 'ip addr', 10_000),
+    );
+    expect(result.exitCode).toBe(124);
+    expect(result.stderr).toContain('timed out after 10s');
+  });
+
+  test('carries a non-zero exit status and stderr through', () => {
+    const failingCli: DockerCli = {
+      exec() {
+        const err = new Error('exited 7') as Error & { status?: number; stderr?: string };
+        err.status = 7;
+        err.stderr = 'no such container';
+        throw err;
+      },
+      spawn() {
+        throw new Error('must not spawn');
+      },
+    };
+    const result = withDockerCli(failingCli, () =>
+      dockerExec(TEST_PROJECT, '/tmp/compose', 'ghost', 'echo hi'),
+    );
+    expect(result.exitCode).toBe(7);
+    expect(result.stderr).toBe('no such container');
+  });
+});
+
+describe('missingImages', () => {
+  test('filters tags docker already has, adding :latest where the caller omitted it', () => {
+    const { cli, commands } = fakeDockerCli({
+      'docker images': 'celilo-e2e/management:latest\ncelilo-e2e/observer:latest\n',
+    });
+    const missing = withDockerCli(cli, () =>
+      missingImages([
+        'celilo-e2e/management',
+        'celilo-e2e/observer:latest',
+        'celilo-e2e/target-machine',
+      ]),
+    );
+    expect(missing).toEqual(['celilo-e2e/target-machine']);
+    expect(commands.length).toBe(1);
+  });
+
+  test('answers [] for an empty ask without asking docker', () => {
+    const { cli, commands } = fakeDockerCli({});
+    expect(withDockerCli(cli, () => missingImages([]))).toEqual([]);
+    expect(commands).toEqual([]);
+  });
+});
+
+describe('scrubDnsZones', () => {
+  test('resolves when the running server serves the expected apex address', async () => {
+    const { cli } = fakeDockerCli({
+      'kdig @127.0.0.1 celilo.computer A +short': `${SIMULATOR_IPS.WEBSITE}\n`,
+    });
+    await expect(withDockerCli(cli, () => scrubDnsZones())).resolves.toBeUndefined();
+  });
+
+  test('throws loudly when the reset did not take effect', async () => {
+    const { cli } = fakeDockerCli({
+      // The reset must succeed so the run reaches the verification query.
+      'cp -f /seed/*.zone': '',
+      'kdig @127.0.0.1 celilo.computer A +short': '203.0.113.9\n',
+    });
+    await expect(withDockerCli(cli, () => scrubDnsZones())).rejects.toThrow(/verification FAILED/);
+  });
+
+  test('a failed reset warns instead of aborting the run', async () => {
+    const warn = console.warn;
+    const warnings: string[] = [];
+    console.warn = (message: unknown) => warnings.push(String(message));
+    try {
+      const deadCli: DockerCli = {
+        exec() {
+          throw new Error('container not running');
+        },
+        spawn() {
+          throw new Error('must not spawn');
+        },
+      };
+      await withDockerCli(deadCli, () => scrubDnsZones());
+      expect(warnings.some((w) => w.includes('[dns-scrub]'))).toBe(true);
+    } finally {
+      console.warn = warn;
+    }
+  });
+});
+
+describe('withDockerCli', () => {
+  test('scopes the override: nested scopes see their own runner, and the outer one resumes', () => {
+    const outer = fakeDockerCli({ 'docker images': 'outer-tag:latest\n' });
+    const inner = fakeDockerCli({ 'docker images': 'inner-tag:latest\n' });
+
+    withDockerCli(outer.cli, () => {
+      expect(missingImages(['outer-tag'])).toEqual([]);
+      withDockerCli(inner.cli, () => {
+        expect(missingImages(['inner-tag'])).toEqual([]);
+      });
+      // The inner scope's fake is gone — only the outer one answers now.
+      // (missingImages answers a docker failure with "all missing", so an
+      // unanswerable tag coming back as itself proves no fake answered.)
+      expect(missingImages(['outer-tag'])).toEqual([]);
+      expect(missingImages(['inner-tag'])).toEqual(['inner-tag']);
+    });
+  });
+
+  test('holds the override across awaits and restores it after', async () => {
+    const fake = fakeDockerCli({ 'docker images': 'async-tag:latest\n' });
+    await withDockerCli(fake.cli, async () => {
+      await Promise.resolve();
+      expect(missingImages(['async-tag'])).toEqual([]);
+    });
+    // Restoration is proven by scoping, not by touching a real daemon: a
+    // fresh override must see no leftover of the previous fake.
+    const probe = fakeDockerCli({ 'docker images': 'probe-tag:latest\n' });
+    withDockerCli(probe.cli, () => {
+      expect(missingImages(['async-tag'])).toEqual(['async-tag']);
+    });
+  });
+
+  test('restores the previous runner even when the callback throws', () => {
+    const fake = fakeDockerCli({ 'docker images': 'boom-tag:latest\n' });
+    expect(() =>
+      withDockerCli(fake.cli, () => {
+        throw new Error('boom');
+      }),
+    ).toThrow('boom');
+    const probe = fakeDockerCli({ 'docker images': 'probe-tag:latest\n' });
+    withDockerCli(probe.cli, () => {
+      expect(missingImages(['boom-tag'])).toEqual(['boom-tag']);
+    });
+  });
+});