@celilo/e2e 0.20.5 → 0.21.0

package/config/ssh/sshd-celilo-api.conf

@@ -0,0 +1,6 @@
+# Remote-API authorisation for the e2e management box. See the COPY of this
+# file in docker/Dockerfile.management for why the account is root here and
+# celilo-api in production.
+Match User root
+  AuthorizedKeysCommand /usr/local/bin/celilo api authorized-keys
+  AuthorizedKeysCommandUser root

package/docker/Dockerfile.management

@@ -124,11 +124,58 @@ ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
 # fall through to Bun's normal resolution.
 RUN printf '[install.scopes]\n"@celilo" = "http://npm-registry.lab"\n' > /root/.bunfig.toml
 
+# --- The fleet browser runtime (managed-browser-runtime) ----------------------
+#
+# Browser-driven health checks (celilo-web-console is the first) resolve
+# `/var/lib/celilo/browsers/current/chrome` and a descriptor describing what
+# was installed. This image provisions that runtime the way the celilo-mgmt
+# role does on a real management host (`modules/celilo-mgmt/ansible/roles/
+# celilo-mgmt/tasks/debian.yml`), pinned to the SAME playwright version that
+# role declares — the fleet's single browser-version declaration. A module
+# bundling a different client is a pin-drift finding, not a failure: the
+# consumer passes the stable path explicitly.
+#
+# Revision comes from the pinned client itself (`install --dry-run`), never
+# from a hand-maintained version→revision table — same reasoning as the role.
+ARG PLAYWRIGHT_VERSION=1.60.0
+RUN set -eux; \
+    REVISION="$(PLAYWRIGHT_BROWSERS_PATH=/var/lib/celilo/browsers bun x "playwright-core@${PLAYWRIGHT_VERSION}" install --dry-run chromium-headless-shell \
+      | grep -oE 'chromium_headless_shell-[0-9]+' | head -1 | cut -d- -f2)"; \
+    PLAYWRIGHT_BROWSERS_PATH=/var/lib/celilo/browsers bun x "playwright-core@${PLAYWRIGHT_VERSION}" install chromium-headless-shell; \
+    PLAYWRIGHT_BROWSERS_PATH=/var/lib/celilo/browsers bun x "playwright-core@${PLAYWRIGHT_VERSION}" install-deps chromium-headless-shell; \
+    BIN="$(find /var/lib/celilo/browsers/chromium_headless_shell-"${REVISION}" -type f \
+      \( -name chrome-headless-shell -o -name headless_shell \) | head -1)"; \
+    test -x "${BIN}"; \
+    mkdir -p /var/lib/celilo/browsers/current; \
+    ln -sf "${BIN}" /var/lib/celilo/browsers/current/chrome; \
+    BROWSER_VERSION="$(/var/lib/celilo/browsers/current/chrome --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)"; \
+    printf '{"schemaVersion":1,"browser":"chromium","flavor":"headless-shell","browserVersion":"%s","playwrightVersion":"%s","revision":"%s","writtenBy":"e2e management image"}' \
+      "${BROWSER_VERSION}" "${PLAYWRIGHT_VERSION}" "${REVISION}" \
+      > /var/lib/celilo/browsers/current/celilo-browser.json; \
+    cat /var/lib/celilo/browsers/current/celilo-browser.json
+
 # Standard modules pre-packaged as .netapp files — built by e2e-build.
 # Tests import from /netapps/<module>.netapp instead of the monorepo mount.
 COPY netapps/ /netapps/
 
 COPY config/ssh/generate-keys.sh /config/ssh/generate-keys.sh
+
+# The remote API's sshd landing config. `celilo --remote` and the web console
+# both reach this box as an ssh forced command, and the keys that may do so
+# come from celilo's own api_principals table AT LOGIN -- never from a static
+# authorized_keys somebody has to keep in sync. A module enrols its principal
+# during its deploy, which is always after this image was built, so a file
+# written at build time could not contain it (celilo#1442).
+#
+# Production lands on the unprivileged `celilo-api` account and the deb's
+# wrapper sudo-drops it into the `celilo` user; the celilo-bootstrap postinst
+# writes the same two directives for it. Here the box was provisioned by
+# install.sh, which installs celilo for root and keeps its database under
+# /root -- there is no `celilo` user to drop into, so root IS the privileged
+# account and the landing account too. The mechanism under test is identical;
+# only the account name differs, which is why `api_dest` is module config.
+COPY config/ssh/sshd-celilo-api.conf /etc/ssh/sshd_config.d/50-celilo-api.conf
+
 COPY config/routing/management-routes.sh /startup.sh
 RUN chmod +x /startup.sh /config/ssh/generate-keys.sh
 

package/docker/Dockerfile.minio

@@ -1,8 +1,19 @@
-# minio — the e2e mirror of AWS S3 (and any S3-compatible backup target).
+# The e2e mirror of AWS S3 (and any S3-compatible backup target).
+#
+# ⚠️ The service, hostname (`minio.lab`) and this file's name are unchanged, but
+# the SERVER IS NO LONGER MinIO. MinIO withdrew public anonymous distribution of
+# its container images: Docker Hub delisted `minio/mc` and `minio/minio`
+# (celilo#1389), the fix moved them to quay.io, and four days later quay.io
+# answered 401 for both the floating tag AND the pinned digest. There was no
+# third registry to move to, so this now runs s3proxy, which implements the S3
+# API over a filesystem backend and is pullable anonymously. The names are kept
+# because renaming the simulator touches the compose generator, simulator-ips,
+# the resolver config and every suite that resolves `minio.lab`; that rename is
+# worth doing and is not worth coupling to an outage fix.
 #
 # Celilo's backup storage provider (apps/celilo/src/services/storage-providers/
 # s3.ts) talks plain S3 with forcePathStyle + a custom endpoint, so it drives
-# MinIO unchanged. This sim is the destination for `celilo backup --storage`
+# this unchanged. This sim is the destination for `celilo backup --storage`
 # (push) and the source for `celilo backup pull` (download) in the migration
 # restore e2e (P5, openspec/specs/management-server-backup/spec.md).
 #
@@ -13,31 +24,38 @@
 # bypasses fw-ext's Squid, so no TLS bump / cert / nonstandard-port filtering).
 #
 # The `celilo-backups` bucket is pre-created at startup (celilo's S3 provider
-# does NOT create buckets — its initialize() only probes). Root creds are the
-# well-known minioadmin/minioadmin; the migration test passes them to
+# does NOT create buckets — its initialize() only probes with ListObjectsV2).
+# Under the filesystem backend a bucket IS a directory, so that is now a plain
+# `mkdir` — no S3 round trip, and no client binary to pull. Root creds are the
+# well-known minioadmin/minioadmin, kept verbatim because
+# e2e/tests/migration-restore.test.ts:40-41 passes them to
 # `celilo storage add s3 --access-key-id ... --secret-access-key ...`.
 
-FROM quay.io/minio/mc:latest@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 AS mc
-
-# Static busybox purely for its `ip` applet: the MinIO image has no iproute2
+# Static busybox purely for its `ip` applet: the s3proxy image has no iproute2
 # and no package manager worth invoking, but it still needs the ISP route
 # every host on internet-external carries (see minio-startup.sh).
 FROM busybox:stable-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 AS busybox
 
-FROM quay.io/minio/minio:latest@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
+FROM andrewgaul/s3proxy@sha256:87662b2a5afcdfa5f478a1c61650bae4c87bbd15f6ff2823236bcfd66ef7fe04
 
 COPY --from=busybox /bin/busybox /usr/local/bin/busybox
 
-# Bring the MinIO client in from the mc image so the entrypoint can create the
-# bucket once the server is up.
-COPY --from=mc /usr/bin/mc /usr/bin/mc
-
 COPY config/routing/minio-startup.sh /usr/local/bin/minio-startup.sh
 
-ENV MINIO_ROOT_USER=minioadmin \
-    MINIO_ROOT_PASSWORD=minioadmin \
+# aws-v2-or-v4 because celilo's SDK signs v4 while older tooling may sign v2;
+# accepting both costs nothing and the alternative is a signature-version
+# mismatch that surfaces as an opaque 403.
+ENV S3PROXY_AUTHORIZATION=aws-v2-or-v4 \
+    S3PROXY_IDENTITY=minioadmin \
+    S3PROXY_CREDENTIAL=minioadmin \
+    S3PROXY_ENDPOINT=http://0.0.0.0:80 \
+    JCLOUDS_PROVIDER=filesystem \
+    JCLOUDS_FILESYSTEM_BASEDIR=/data \
     MINIO_BUCKET=celilo-backups
 
-EXPOSE 80 9001
+EXPOSE 80
 
-ENTRYPOINT ["/bin/sh", "/usr/local/bin/minio-startup.sh"]
+# dumb-init stays the entrypoint so signals reach the server; the startup script
+# takes CMD and execs s3proxy's own launcher at the end.
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/bin/sh", "/usr/local/bin/minio-startup.sh"]

package/docker/Dockerfile.socks-proxy

@@ -18,6 +18,20 @@ RUN apt-get update && apt-get install -y \
     iproute2 \
     iputils-ping \
     dnsutils \
+    # curl: the proxy is not only a SOCKS daemon, it is a VANTAGE. Tests reach
+    # the deployed app "as a user inside the simulation" by running a client
+    # INSIDE this container (SocksProxyHandle.exec is `docker exec <proxy> bash
+    # -c ...`, socks-proxy.ts:234), so whatever the test runs has to exist here.
+    # Without curl, `proxy.exec("curl ... -w '%{http_code}'")` produced an EMPTY
+    # string, and an assertion reading only stdout cannot tell "the server
+    # returned nothing" from "I could not run the client". celilo#1426's stage 6
+    # failed that way and had never once passed.
+    curl \
+    # wireguard-tools: the control-plane-vpn vantage. A browser that reaches
+    # celilo-web-console (secure-mgmt, VPN-only per D7) must enter as a real
+    # VPN peer — the proxy brings up a wg tunnel, it does not attach directly
+    # to the zone. Needs /dev/net/tun at `docker run` time (socks-proxy.ts).
+    wireguard-tools \
     && rm -rf /var/lib/apt/lists/*
 
 COPY config/socks/danted.conf /etc/danted.conf

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.20.5",
+  "version": "0.21.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -38,7 +38,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@celilo/capabilities": "^5.1.1",
+    "@celilo/capabilities": "^6.1.0",
     "@celilo/cli-display": "^0.2.0",
     "@celilo/event-bus": "^0.7.0",
     "@celilo/terraform-fake": "^0.3.1",

package/registry-server/src/bootstrap.ts

@@ -16,7 +16,14 @@
  */
 
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  writeFileSync,
+} from 'node:fs';
 import { basename, dirname, join, relative } from 'node:path';
 import type { IndexEntry } from './storage';
 
@@ -104,18 +111,42 @@ export function scanUploadsDir(uploadsDir: string): Map<string, BootstrapEntry>
   return entries;
 }
 
-/** Render a bootstrap entry as an IndexEntry suitable for the sparse index. */
-export function bootstrapIndexEntry(entry: BootstrapEntry): IndexEntry {
+/**
+ * Render a bootstrap entry as an IndexEntry suitable for the sparse index.
+ *
+ * `cksum` defaults to the literal sentinel `bootstrap`, which the client reads
+ * as "no integrity data" and skips (apps/celilo/src/registry/client.ts
+ * isVerifiableCksum). Pass the real digest whenever it is knowable — see
+ * `bootstrapCksum` — so the consumer's integrity check actually runs.
+ */
+export function bootstrapIndexEntry(entry: BootstrapEntry, cksum = 'bootstrap'): IndexEntry {
   const vers = entry.version.includes('+') ? entry.version : `${entry.version}+1`;
   return {
     name: entry.name,
     vers,
     deps: [],
-    cksum: 'bootstrap',
+    cksum,
     yanked: false,
   };
 }
 
+/**
+ * sha256 of the exact .netapp bytes this entry serves, packaging it if it is
+ * not cached yet.
+ *
+ * The bootstrap path packages on demand, so it holds the bytes at the moment
+ * it serves them — the digest IS knowable, it was just never computed
+ * (celilo#1262). Publishing it means every e2e import exercises the download
+ * integrity check instead of skipping it on the sentinel, and a corrupt
+ * bootstrap artifact is caught at the consumer rather than surfacing later as
+ * "zlib: unexpected end of file".
+ */
+export async function bootstrapCksum(entry: BootstrapEntry, cacheDir: string): Promise<string> {
+  const netappPath = await packageBootstrapModule(entry, cacheDir);
+  const bytes = await Bun.file(netappPath).arrayBuffer();
+  return createHash('sha256').update(new Uint8Array(bytes)).digest('hex');
+}
+
 /**
  * Walk a directory, skipping hidden entries, *.test.ts files, the
  * module's e2e/ tree (tests + their deps don't ship with the module),
@@ -238,11 +269,18 @@ export async function packageBootstrapModule(
     }
     writeFileSync(join(stagingDir, 'checksums.json'), checksumsJson);
 
-    const proc = Bun.spawn(['tar', '-czf', cachePath, '-C', stagingDir, '.'], {
+    // Write to a temp path and rename into place. The cached artifact is now
+    // hashed and published as the entry's cksum, so a concurrent reader that
+    // catches a half-written tar would fail its integrity check for a reason
+    // that has nothing to do with the download. Rename is atomic on the same
+    // filesystem, so a reader sees either no cache or the whole artifact.
+    const tmpPath = `${cachePath}.${process.pid}.${Date.now()}.tmp`;
+    const proc = Bun.spawn(['tar', '-czf', tmpPath, '-C', stagingDir, '.'], {
       stderr: 'inherit',
     });
     const exitCode = await proc.exited;
     if (exitCode !== 0) throw new Error(`tar exited with code ${exitCode}`);
+    renameSync(tmpPath, cachePath);
   } finally {
     try {
       Bun.spawnSync(['rm', '-rf', stagingDir]);

package/registry-server/src/index.ts

@@ -4,6 +4,7 @@ export { TokenAuth } from './auth';
 export { startServer, startFromEnv } from './server';
 export {
   scanBootstrapDir,
+  bootstrapCksum,
   bootstrapIndexEntry,
   packageBootstrapModule,
 } from './bootstrap';

package/registry-server/src/server.ts

@@ -24,6 +24,7 @@ import { join } from 'node:path';
 import { ADMIN_SCOPE, TokenAuth } from './auth';
 import {
   type BootstrapEntry,
+  bootstrapCksum,
   bootstrapIndexEntry,
   packageBootstrapModule,
   scanBootstrapDir,
@@ -263,6 +264,22 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
     return b ? [bootstrapIndexEntry(b)] : [];
   }
 
+  /**
+   * Same resolution as `resolveEntries`, but a bootstrap-served entry carries
+   * the real sha256 of the .netapp it will serve rather than the `bootstrap`
+   * sentinel (celilo#1262). Packaging happens here so the digest is knowable;
+   * only the sparse-index endpoint needs it, and only that endpoint pays for
+   * it — the search and metadata endpoints never read `cksum`.
+   */
+  async function resolveIndexEntries(name: string): Promise<IndexEntry[]> {
+    if (!isValidName(name)) return [];
+    const published = storage.readIndex(name);
+    if (published.length > 0) return published;
+    const b = resolveBootstrap().get(name);
+    if (!b) return [];
+    return [bootstrapIndexEntry(b, await bootstrapCksum(b, bootstrapCacheDir))];
+  }
+
   async function serveBootstrapDownload(entry: BootstrapEntry): Promise<Response> {
     const netappPath = await packageBootstrapModule(entry, bootstrapCacheDir);
     return new Response(Bun.file(netappPath), {
@@ -542,7 +559,7 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
       if (method === 'GET' && suffix.startsWith('/index/')) {
         const name = decodeURIComponent(suffix.slice(suffix.lastIndexOf('/') + 1));
         if (!isValidName(name)) return notFound();
-        const entries = resolveEntries(name);
+        const entries = await resolveIndexEntries(name);
         if (entries.length === 0) return notFound();
         const body = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
         return new Response(body, {

package/simulators/greenwave/state.ts

@@ -6,8 +6,68 @@ let validUsername = process.env.ROUTER_USERNAME || 'admin';
 let validPassword = process.env.ROUTER_PASSWORD || 'admin';
 const PUBLIC_IP = process.env.ROUTER_PUBLIC_IP || '203.0.113.100';
 
-// The network interface facing the ISP (isp-external network)
-const EXTERNAL_INTERFACE = process.env.EXTERNAL_INTERFACE || 'eth1';
+/**
+ * The interface facing the ISP — DERIVED from the address it has to serve, not
+ * guessed.
+ *
+ * The port-forward DNAT used to bind an interface (`-i ${EXTERNAL_INTERFACE}`),
+ * defaulting to `eth1`, and nothing anywhere set that variable — not the
+ * compose generator, not any config. On fw-isp `eth1` is the INTERNAL leg
+ * (10.226.1.1/24) and `eth0` carries the public address, so every forward was
+ * bound to the interface facing AWAY from the internet.
+ *
+ * What that costs, measured 2026-09-26: inbound 80/443 to 203.0.113.100 was
+ * refused, so Pebble could complete NEITHER http-01 (`dial 203.0.113.100:80:
+ * connect: connection refused`) nor tls-alpn-01 (`Failed to connect to
+ * 203.0.113.100:443`). No per-test domain could ever obtain a certificate.
+ * celilo-web-console-oidc died at stage 2 on authentik's HTTPS readiness and
+ * again at stage 4 on OIDC discovery — two failures, one cause, neither of them
+ * naming the interface. Adding the eth0 rule by hand made ACME finish in 12s.
+ *
+ * Hairpin traffic from inside DID match, so forwards appeared to work from
+ * every vantage inside the simulation. Only the internet could not get in,
+ * which is the one direction a port forward exists for.
+ *
+ * Deriving it removes the class: whichever interface holds PUBLIC_IP is by
+ * definition the one inbound traffic arrives on. The env var still wins when
+ * set, and the old default remains the last resort so a container whose address
+ * is not yet up behaves as before rather than throwing.
+ */
+function interfaceCarrying(address: string): string | null {
+  try {
+    const out = execSync(`ip -4 -o addr show | grep -w ${address} || true`, {
+      encoding: 'utf8',
+      timeout: 5_000,
+    });
+    // `ip -o addr` columns: "<idx>: <iface> inet <addr>/<len> ..."
+    return out.trim().split('\n')[0]?.trim().split(/\s+/)[1] || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The interface the DHCP server must bind, derived from the router address it
+ * hands out rather than assumed to be eth0.
+ *
+ * dnsmasq is configured `bind-interfaces`, so this decides which segment it
+ * serves. fw-isp is multi-homed and Docker's interface ordering is NOT stable
+ * across runs — measured 2026-09-26, eth0 and eth1 swapped between two runs on
+ * the same rig. Bound to the wrong one, dnsmasq offers a 10.226.1.0/24 range on
+ * the PUBLIC segment and serves nothing on the LAN it exists for, which is a
+ * worse failure than not starting.
+ *
+ * Same defect as the port-forward DNAT above, same file, found by auditing for
+ * it after that one bit twice in opposite directions.
+ */
+function deriveDhcpInterface(routerIp: string): string {
+  const iface = interfaceCarrying(routerIp);
+  if (iface) return iface;
+  console.error(
+    `[greenwave] no interface carries the DHCP router address ${routerIp}; binding eth0 and DHCP may serve the wrong segment`,
+  );
+  return 'eth0';
+}
 
 /**
  * Vendor extension prefix for TR-181 parameters.
@@ -201,10 +261,11 @@ function rewriteDnsmasqConfig(): void {
   const dnsServers = dhcpPool.DNSServers || '10.226.1.1';
   const domainName = dhcpPool.DomainName || '';
 
+  const routerIp = dhcpPool.IPRouters || '10.226.1.1';
   const lines = [
     '# Auto-generated by greenwave simulator',
     'port=0',
-    'interface=eth0',
+    `interface=${deriveDhcpInterface(routerIp)}`,
     'bind-interfaces',
     'dhcp-range=10.226.1.200,10.226.1.220,255.255.255.0,60s',
     'dhcp-option=option:router,10.226.1.1',
@@ -244,7 +305,27 @@ function applyIptablesRule(mapping: PortMapping): void {
   function addRules(p: string) {
     // DNAT incoming traffic on external interface destined for our public IP
     execSync(
-      `iptables -t nat -A PREROUTING -i ${EXTERNAL_INTERFACE} -d ${PUBLIC_IP} -p ${p} --dport ${extPort} -j DNAT --to-destination ${intIp}:${intPort}`,
+      // NO `-i` HERE, DELIBERATELY. The destination address IS the
+      // discriminator: only traffic actually addressed to PUBLIC_IP can match,
+      // whichever leg it arrives on. Binding an interface as well makes exactly
+      // one direction work and silently breaks the other.
+      //
+      // Measured 2026-09-26, and it is a COIN FLIP because Docker's interface
+      // ordering is not stable across runs. Same rule, two runs:
+      //   run24/25  eth0=203.0.113.100 (public), eth1=10.226.1.1
+      //             -> `-i eth1` matched hairpin only; INBOUND refused, so no
+      //                per-test domain could obtain a certificate at all.
+      //   run28     eth0=10.226.1.1, eth1=203.0.113.100 (public)
+      //             -> `-i eth1` matched inbound only; HAIRPIN refused, so the
+      //                SOCKS browser got ERR_SOCKS_CONNECTION_FAILED reaching
+      //                the public login path from inside the simulation.
+      // Verified on the live stack with the rule as it stood: from the internal
+      // proxy REFUSED, from Pebble on internet-external OPEN, simultaneously.
+      //
+      // Deriving the "right" interface does not fix this — it only chooses
+      // which direction breaks. Both are real paths: ACME comes in from the
+      // internet, and a browser inside the simulation dials the public name.
+      `iptables -t nat -A PREROUTING -d ${PUBLIC_IP} -p ${p} --dport ${extPort} -j DNAT --to-destination ${intIp}:${intPort}`,
       { timeout: 5000 },
     );
     // Allow forwarded traffic

package/src/bake-probe-explanation.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * The bake probe waits for HTTPS to VERIFY, so a trust failure and an
+ * unfinished issuance both spend the whole budget and both used to print
+ * "timed out" — which is why celilo#1365 and celilo#1427 read as timeouts for
+ * weeks while `curl -k` returned 200 throughout. They need opposite responses,
+ * so the headline must branch on curl's exit code.
+ *
+ * The explainer lives in a bin script (no import surface), so this gate reads
+ * the real source and pins the behaviour that matters: the trust codes and the
+ * reach codes are classified, disjointly, and the trust branch says waiting
+ * cannot help.
+ */
+describe('bake probe explains which failure it hit', () => {
+  const src = readFileSync(join(import.meta.dir, '..', 'bin', 'e2e-bake-management'), 'utf-8');
+  const fn = /function explainProbeFailure\(curlExit: number\): string \{[\s\S]*?\n\}/.exec(src);
+
+  test('the explainer exists', () => {
+    expect(fn).not.toBeNull();
+  });
+
+  const body = fn?.[0] ?? '';
+
+  test('trust codes and reach codes are classified disjointly', () => {
+    const arrays = [...body.matchAll(/\[([\d,\s]+)\]\.includes\(curlExit\)/g)].map((m) =>
+      m[1].split(',').map((n) => Number(n.trim())),
+    );
+    expect(arrays.length).toBe(2);
+    const [trust, reach] = arrays;
+    // 60 unknown issuer is the dead-root signature; 6 unresolved is the reach one.
+    expect(trust).toContain(60);
+    expect(reach).toContain(6);
+    expect(trust.filter((c) => reach.includes(c))).toEqual([]);
+  });
+
+  test('the trust branch says waiting cannot fix it', () => {
+    const trustBranch = body.slice(0, body.indexOf('6, 7, 28'));
+    expect(trustBranch).toMatch(/waiting longer cannot fix it/i);
+  });
+});

package/src/bake-probe-fails-closed.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * The HTTPS probe must FAIL CLOSED when it cannot read curl's exit code.
+ *
+ * Its command ends in `echo "curl-exit=$?"`, so the shell always exits 0 — a
+ * fallback to the exec's own exit code therefore scores "I could not read the
+ * answer" as "the site is serving". Measured on run 7258: the probe printed
+ * `pebble CA ready ✔ 0s` while celilo.computer served NO certificate at all
+ * (empty `openssl s_client` chain, server aborting with `internal error
+ * (592)`), and the real defect surfaced one step later as install.sh's
+ * curl (35). A probe that cannot fail turned celilo#1365 into a green step.
+ *
+ * Same shape as the bin script's other gate: the explainer has no import
+ * surface, so this reads the real source.
+ */
+describe('the bake HTTPS probe fails closed', () => {
+  const src = readFileSync(join(import.meta.dir, '..', 'bin', 'e2e-bake-management'), 'utf-8');
+
+  test('an unreadable probe does NOT fall back to the shell exit code', () => {
+    expect(src).not.toContain('Number(code[1]) : probe.exitCode');
+  });
+
+  test('an unreadable probe is scored as not-ready', () => {
+    expect(src).toContain('Number(code[1]) : PROBE_UNREADABLE');
+  });
+
+  test('the unreadable case has its own explanation, not the timeout headline', () => {
+    const fn = /function explainProbeFailure\(curlExit: number\): string \{[\s\S]*?\n\}/.exec(src);
+    expect(fn).not.toBeNull();
+    expect(fn?.[0]).toContain('PROBE_UNREADABLE');
+  });
+});

package/src/bake-probe-no-shell-expansion.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * No command this script sends into a container may use `$?`.
+ *
+ * dockerExec wraps the command as `bash -c ${JSON.stringify(cmd)}` and hands
+ * that to a HOST shell (container-manager.ts:636), so a `$?` inside it is
+ * expanded by the host BEFORE the container sees it — to the host's own last
+ * status, which is 0.
+ *
+ * Measured on celilo-builder run 7262, where one layer printed both lines:
+ *
+ *   curl: (60) SSL certificate problem: unable to get local issuer certificate
+ *   curl-exit=0
+ *
+ * The bake's HTTPS probe read that 0 and passed. It had passed on every run in
+ * celilo#1365's history — first hiding the defect behind a 30s timeout, later
+ * behind a green step and an unexplained install.sh failure. A probe that
+ * cannot fail is not a probe, and the failure it hid was the real one.
+ *
+ * curl's own `%{exitcode}` carries the status with no shell variable to expand.
+ */
+describe('no container command relies on host-expanded shell variables', () => {
+  const src = readFileSync(join(import.meta.dir, '..', 'bin', 'e2e-bake-management'), 'utf-8');
+  // Comments explain the hazard and must be allowed to name it.
+  const code = src
+    .split('\n')
+    .filter((line) => !line.trim().startsWith('//') && !line.trim().startsWith('*'))
+    .join('\n');
+
+  test('`$?` appears in no executed command', () => {
+    expect(code).not.toContain('$?');
+  });
+
+  test("the probe reads curl's exit code from curl itself", () => {
+    expect(code).toContain('curl-exit=%{exitcode}');
+  });
+});

package/src/cli/build.test.ts

@@ -12,10 +12,12 @@ import {
 import { tmpdir } from 'node:os';
 import { basename, join } from 'node:path';
 import { gzipSync } from 'node:zlib';
+import { readDockerfileBases } from '../doctor';
 import {
   assertGzipValid,
   bakeManagement,
   packageNetapp,
+  pullDockerfileBases,
   reportBakeChildFailure,
   stageNetappsFromRegistry,
   verifyNetapp,
@@ -234,3 +236,51 @@ test('bakeManagement classifies a real child exit 3 end to end and exits 3', ()
     rmSync(pkgDir, { recursive: true, force: true });
   }
 });
+
+/**
+ * Reach probe for the base-image pull (celilo#1244).
+ *
+ * `build-infra` is the one-command remedy doctor's base-images warn names, so
+ * it has to reach EVERY base, and the set it walks is computed — a directory
+ * walk over `docker/Dockerfile.*`. Reasoning about that walk is what let the
+ * old check skip ten images, so measure it instead: mirror the REAL Dockerfiles
+ * (never a hand-written list, which would give confident reach data about a
+ * directory that does not exist), put a recording shim on PATH in place of
+ * docker, and read back which refs it was actually asked to pull.
+ */
+test('build-infra pulls every base image a real Dockerfile names', () => {
+  const pkgDir = join(import.meta.dir, '..', '..');
+  const expected = [...new Set(readDockerfileBases(pkgDir).flatMap((d) => d.bases))].sort();
+  expect(expected.length).toBeGreaterThan(0);
+
+  // Mirror the real docker/ dir so the walk has the real shape to walk.
+  const mirror = mkdtempSync(join(tmpdir(), 'base-reach-'));
+  mkdirSync(join(mirror, 'docker'));
+  for (const f of readdirSync(join(pkgDir, 'docker')).filter((f) => f.startsWith('Dockerfile.'))) {
+    writeFileSync(join(mirror, 'docker', f), readFileSync(join(pkgDir, 'docker', f), 'utf-8'));
+  }
+
+  const log = join(mirror, 'pulls.txt');
+  const shimDir = join(mirror, 'bin');
+  mkdirSync(shimDir);
+  writeFileSync(
+    join(shimDir, 'docker'),
+    `#!/bin/sh
+[ "$1" = pull ] && printf '%s\n' "$3" >> ${log}
+exit 0
+`,
+    { mode: 0o755 },
+  );
+
+  const savedPath = process.env.PATH;
+  process.env.PATH = `${shimDir}:${savedPath}`;
+  try {
+    expect(pullDockerfileBases(mirror)).toEqual([]);
+  } finally {
+    process.env.PATH = savedPath;
+  }
+
+  const pulled = readFileSync(log, 'utf-8').trim().split('\n').sort();
+  expect(pulled).toEqual(expected);
+  rmSync(mirror, { recursive: true, force: true });
+});