@celilo/e2e 0.20.5 → 0.21.0

package/bin/e2e-bake-management

@@ -24,14 +24,22 @@
 
 import { execSync } from 'node:child_process';
 import { join } from 'node:path';
-import { ensureSharedInfra } from '../src/shared-infra';
+import { ensureSharedInfra, stopSharedInfra } from '../src/shared-infra';
 import { network } from '../src/network-builder';
 import { imageTag } from '../src/docker-compose-generator';
 import { findMonorepoRoot } from '../src/repo-root';
+import { detectStaleAcmeRoot, explainStaleRoot } from '../src/stale-acme-root';
 import {
   packNpmRegistryTarballs,
   stageWebsiteDist,
 } from '../src/stage-simulator-inputs';
+import {
+  INSTALLED_CLI_DIR,
+  STAGED_TARBALL,
+  digestScript,
+  mismatchMessage,
+  parseDigests,
+} from '../src/installed-cli-digest';
 import {
   CONSUMER_FINGERPRINT_PREFIX,
   PUBLISHED_FINGERPRINT_PREFIX,
@@ -221,6 +229,43 @@ function labelSafeVersion(version: string): string {
   return version.trim().split(/\s+/).pop() || 'unknown';
 }
 
+/**
+ * The staged tarball for a package, as pack-celilo-packages recorded it.
+ */
+function stagedTarballPath(packageName: string): string {
+  const manifestPath = join(PACKAGE_ROOT, '.npm-registry-cache', 'pack-manifest.json');
+  const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')) as {
+    packages?: Array<{ name: string; tarball: string }>;
+  };
+  const entry = manifest.packages?.find((p) => p.name === packageName);
+  if (!entry) {
+    throw new Error(`pack-manifest.json lists no ${packageName} tarball to verify the install against.`);
+  }
+  // pack() records the basename; the tarballs sit beside the manifest.
+  return join(PACKAGE_ROOT, '.npm-registry-cache', entry.tarball);
+}
+
+/**
+ * Refuse to stamp a freshness label over a CLI that is not the one we packed.
+ * See src/installed-cli-digest.ts for why the label alone cannot say this.
+ */
+async function verifyInstalledCliIsStaged(exec: Exec, containerId: string): Promise<void> {
+  process.stdout.write('  installed CLI = staged   ');
+  const t = Date.now();
+  run(`docker cp ${JSON.stringify(stagedTarballPath('@celilo/cli'))} ${containerId}:${STAGED_TARBALL}`);
+  const b64 = Buffer.from(digestScript(INSTALLED_CLI_DIR)).toString('base64');
+  const probe = await exec(
+    `echo ${b64} | base64 -d > /tmp/verify-cli.sh && bash /tmp/verify-cli.sh 2>&1`,
+    120_000,
+  );
+  const digests = parseDigests(`${probe.stdout}${probe.stderr}`);
+  if (digests.staged !== digests.installed) {
+    console.log('\u2717');
+    throw new Error(mismatchMessage(digests));
+  }
+  console.log(`\u2714 ${Math.round((Date.now() - t) / 1000)}s  (${digests.staged.slice(0, 16)})`);
+}
+
 function sourceStamp(published: boolean, version: string): string {
   if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${labelSafeVersion(version)}`;
   // Consumer mode: restageSimulatorInputs is the only writer of
@@ -280,6 +325,148 @@ function restageSimulatorInputs(): void {
  * Default bake: install.sh against the simulated celilo.computer / npm
  * registry, inside the standard topology.
  */
+/**
+ * The probe waits for HTTPS to VERIFY, so a trust failure and an unfinished
+ * issuance both spend the whole budget and both used to print "timed out".
+ * They need opposite responses, so branch the headline on curl's own exit code
+ * rather than making the reader infer it from the layers (crew-builder,
+ * celilo#1427).
+ */
+/**
+ * The TLS/DNS layers, read from inside the management container.
+ *
+ * Hoisted out of the probe's failure path because the probe is not the only
+ * step that can fail on TLS: install.sh's own curl-pipe-bash talks to the same
+ * host a moment later, and when THAT failed the bake printed the curl error
+ * alone with no layers under it (measured, run 7255: `curl: (35) TLS connect
+ * error ... tlsv1 alert internal error`, one second after the probe had
+ * verified the same URL successfully). A failure with no layers is the exact
+ * shape celilo#1365 spent two weeks in.
+ */
+async function tlsDiagnosticLayers(exec: Exec, projectName?: string): Promise<string> {
+  const diag = async (label: string, cmd: string): Promise<string> => {
+    const out = await exec(cmd, 10_000);
+    return `--- ${label} ---\n${out.stdout.trim() || '(no output)'}`;
+  };
+  return [
+    await diag('resolv.conf', 'cat /etc/resolv.conf'),
+    await diag('dig @10.226.1.10 (internal resolver)', 'dig +time=2 +tries=1 @10.226.1.10 celilo.computer'),
+    await diag('dig @203.0.113.1 (public fallback)', 'dig +time=2 +tries=1 @203.0.113.1 celilo.computer'),
+    await diag('trusted CA files', 'ls -la /usr/local/share/ca-certificates/'),
+    await diag('Pebble runtime root reachable from management', 'curl -sk --max-time 5 https://100.64.0.100:15000/roots/0 | head -c 300; echo'),
+    await diag('fetched runtime root (if any)', 'openssl x509 -in /usr/local/share/ca-certificates/pebble-acme-root.crt -noout -subject -enddate 2>/dev/null'),
+    await diag('cert chain served by website-sim', 'timeout 8 openssl s_client -connect celilo.computer:443 -servername celilo.computer -showcerts </dev/null 2>/dev/null | grep -E "^ [0-9] s:|^   i:"'),
+    await diag('curl -v (DNS/connect/TLS/HTTP stages)', 'curl -v --max-time 5 https://celilo.computer/install.sh -o /dev/null 2>&1 | tail -20'),
+    // The probe's EXACT command, re-run here. Runs 7255-7260 all showed the
+    // probe returning 0 and install.sh's curl failing 130ms later on the same
+    // host, which is either two different answers from one server or a probe
+    // reading something other than what it thinks. Printing both side by side
+    // decides it, instead of reasoning about which flag matters.
+    await diag(
+      're-run of the probe command',
+      "curl -sS --max-time 3 -o /dev/null -w 'curl-exit=%{exitcode}\n' https://celilo.computer/install.sh 2>&1",
+    ),
+    await diag(
+      'the same URL with -f, as install.sh fetches it',
+      "curl -fsS --max-time 5 -o /dev/null -w 'curl-exit=%{exitcode}\n' https://celilo.computer/install.sh 2>&1",
+    ),
+    // Did management's own startup script reach its Pebble-root fetch at all?
+    // management-routes.sh runs under `set -e` with six `ip route add` calls
+    // AHEAD of that fetch (line 61), and a route add exiting 2 has killed this
+    // script before — celilo#1351, recorded in docker-compose-generator.ts. If
+    // the trust store has no runtime root, this log says whether the fetch
+    // failed, is still looping, or was never reached.
+    await diag('is /startup.sh still running?', 'ps -ef | grep -c "[s]tartup.sh"'),
+    managementStartupLog(projectName),
+    // Every layer above is read from INSIDE management, so all of them describe
+    // the client. When the server aborts the handshake before sending a
+    // certificate (run 7258), the client has nothing left to report and the
+    // reason is only in Caddy's own log — whether it got an ACME cert at all.
+    websiteSimLog(),
+  ].join('\n');
+}
+
+/** The per-test management container's own startup output, read from the host. */
+function managementStartupLog(projectName?: string): string {
+  if (!projectName) return '--- management startup log ---\n(project name unavailable)';
+  try {
+    const id = execSync(`docker ps -aq --filter name=${projectName} --filter name=management`, {
+      encoding: 'utf-8',
+      timeout: 15_000,
+    })
+      .trim()
+      .split('\n')[0];
+    if (!id) return '--- management startup log ---\n(container not found)';
+    const out = execSync(`docker logs --tail 40 ${id} 2>&1`, {
+      encoding: 'utf-8',
+      timeout: 20_000,
+    });
+    return `--- management startup log (server side) ---\n${out.trim() || '(no output)'}`;
+  } catch (err) {
+    return `--- management startup log ---\n(unreadable: ${(err as Error).message})`;
+  }
+}
+
+/** celilo-website-sim's own Caddy log, read from the host. */
+function websiteSimLog(): string {
+  try {
+    const out = execSync(
+      'docker compose -f docker-compose.shared.yml -p celilo-e2e-shared logs --tail 40 celilo-website 2>&1',
+      // The package's OWN root, not the monorepo's: npm-consumer-smoke runs this
+      // from node_modules outside any checkout, where findMonorepoRoot() is
+      // undefined and join() throws (measured, run 7260). Same resolution
+      // shared-infra.ts:getE2eDir uses, for the same reason.
+      { cwd: join(import.meta.dir, '..'), encoding: 'utf-8', timeout: 20_000 },
+    );
+    return `--- celilo-website-sim log (server side) ---\n${out.trim() || '(no output)'}`;
+  } catch (err) {
+    return `--- celilo-website-sim log (server side) ---\n(unreadable: ${(err as Error).message})`;
+  }
+}
+
+/** Sentinel: the probe ran but its `curl-exit=` marker was not in the output. */
+const CA_WAIT_SECONDS = 240;
+const PROBE_UNREADABLE = -2;
+
+function explainProbeFailure(curlExit: number): string {
+  // 60 unknown issuer, 35 TLS connect, 51 peer cert not ok.
+  if ([60, 35, 51].includes(curlExit)) {
+    return (
+      `celilo.computer served a certificate this container cannot verify (curl ${curlExit}). ` +
+      "The step above already waited for management's runtime root, so this is the STALE-ROOT " +
+      'case, not the not-yet-fetched one — those need opposite responses and exit 60 alone ' +
+      'cannot tell them apart (celilo#1365 spent runs on the wrong one). ' +
+      'This is NOT a timeout and waiting longer cannot fix it. Pebble mints a random root CA ' +
+      'at every startup and Caddy caches its cert until expiry, so a Pebble that restarted ' +
+      'without the website sim restarting is serving a cert from a root that no longer exists ' +
+      '(celilo#1427). Check the chain above against the live root. Remedy: `cele2e down` then ' +
+      '`cele2e build-infra`, so Pebble and Caddy start together.'
+    );
+  }
+  // 6 could not resolve, 7 could not connect, 28 operation timeout.
+  if ([6, 7, 28].includes(curlExit)) {
+    return (
+      `celilo.computer could not be reached at all (curl ${curlExit}) within 30s — a resolution ` +
+      'or connection failure, not a trust one. Either the website sim is down or its ACME ' +
+      'issuance genuinely has not finished. Read the DNS layers above.'
+    );
+  }
+  if (curlExit === PROBE_UNREADABLE) {
+    return (
+      'The HTTPS probe never returned a readable curl exit code in 30s — its ' +
+      '`curl-exit=` marker was missing from both streams. Treated as NOT ready: ' +
+      'the probe cannot prove the site is serving, and a probe that cannot fail ' +
+      'is how celilo#1365 stayed invisible. See the layers above.'
+    );
+  }
+  return (
+    `Timed out (${Math.round(CA_WAIT_SECONDS)}s of wall clock) waiting for celilo.computer to serve install.sh over HTTPS (curl ${curlExit}). ` +
+    'See the DNS / curl -v layers above for which one failed. A cold shared-infra ' +
+    'caddy needs ~90s to obtain its FIRST certificate, so this is a real failure ' +
+    'rather than an impatient probe.'
+  );
+}
+
 async function bakeViaSim(): Promise<void> {
   console.log('▸ Baking celilo into management:latest via install.sh');
   console.log('');
@@ -302,30 +489,146 @@ async function bakeViaSim(): Promise<void> {
   const exec: Exec = (cmd, timeoutMs) => net.exec('management', cmd, timeoutMs);
 
   let containerId = '';
+  let bakeFailed = false;
   try {
     // Wait for the management container's Pebble CA fetch to complete so
     // the curl-pipe-bash below can verify TLS to celilo.computer. Without
     // pipefail an early curl failure returns 0 silently and install.sh
     // "runs" in 0s installing nothing.
+    // WAIT FOR THE DEPENDENCY THIS STEP IS NAMED AFTER, not for a symptom of it.
+    // management-routes.sh fetches Pebble's runtime root in a FOREGROUND retry
+    // loop of 60 tries x 2s (config/routing/management-routes.sh:61-71) — worst
+    // case 120s, and it only starts once the per-test routing is up. The HTTPS
+    // probe below budgeted 30s, so it gave up while that fetch was still
+    // running and blamed the certificate. Measured on run 7266: /startup.sh
+    // still running, its log stopped at "Waiting for Pebble ACME root CA..."
+    // with no "installed" line, and the trust store held only build-time certs
+    // — while the website sim had already obtained a valid cert. The two waits
+    // were simply sized wrong relative to each other.
+    process.stdout.write('  pebble root installed .... ');
+    const tRoot = Date.now();
+    let rootReady = false;
+    // 240s, not the 150 this was first written with. MEASURED on run 7268:
+    // the fetch took 136s — past the loop's nominal 120s ceiling, because each
+    // failing curl spends its own connect timeout before the 2s sleep. 150
+    // left 10% headroom on an idle builder, which is how a green gate starts
+    // flaking under load. The budget costs nothing when the root arrives fast:
+    // the wait ends the moment the file appears.
+    for (let i = 0; i < 240; i++) {
+      const probe = await exec(
+        'test -s /usr/local/share/ca-certificates/pebble-acme-root.crt && echo root-present || echo root-absent',
+        5_000,
+      );
+      if (`${probe.stdout}${probe.stderr}`.includes('root-present')) {
+        rootReady = true;
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+    if (!rootReady) {
+      console.log('✗');
+      console.error(await tlsDiagnosticLayers(exec, net.projectName));
+      throw new Error(
+        "management never installed Pebble's runtime ACME root in 240s. Its startup script " +
+          'fetches it in a foreground loop before celilo.computer can be verified at all; the ' +
+          'layers above say whether that loop is still running, died early, or is failing its ' +
+          'fetch (config/routing/management-routes.sh:61).',
+      );
+    }
+    console.log(`✔ ${Math.round((Date.now() - tRoot) / 1000)}s`);
+
     process.stdout.write('  pebble CA ready .......... ');
     const tCa = Date.now();
     let caReady = false;
-    for (let i = 0; i < 30; i++) {
+    let lastProbeExit = -1;
+    let lastProbeOutput = '';
+    // Re-checked while INCONCLUSIVE, at most every 30s. A single check at t=0
+    // sits at the flakiest moment in the run (shared infra has just come up), and
+    // an exec that answers nothing there scores `unknown` — so a check that fired
+    // only once would silently fall back to burning the whole budget, which is the
+    // exact defect this is here to remove.
+    let staleCheckAt = 0;
+    // WALL CLOCK, not an iteration count. Each pass costs up to 3s of curl plus a
+    // 1s sleep, so `i < 30` was never 30 seconds — and the error text said "30s"
+    // regardless, which is how this budget got read as generous when it was not.
+    // Measured 2026-09-26 on a torn-down host: shared-infra caddy needs ~90s to
+    // obtain its FIRST certificate (caddy's own log: "certificate obtained
+    // successfully" at 90.9s elapsed, after one tls-alpn-01 retry). A 30-iteration
+    // cap could not reach that, so a cold rig failed the bake three runs running
+    // while the website was merely still issuing. 240s covers the measurement with
+    // margin and still fails in well under the suite's own budget.
+    const CA_WAIT_MS = 240_000;
+    const caDeadline = Date.now() + CA_WAIT_MS;
+    while (Date.now() < caDeadline) {
+      // NO `$?` HERE, EVER. dockerExec wraps the command as
+      // `bash -c ${JSON.stringify(cmd)}` (container-manager.ts:636) and hands
+      // that to a HOST shell, which expands `$?` itself — to its own last
+      // status, which is 0. So `echo "curl-exit=$?"` reported curl-exit=0 on
+      // every run this bead has ever had, including the ones where curl printed
+      // `(60) SSL certificate problem: unable to get local issuer certificate`
+      // two lines above it (run 7262 prints exactly that pair). The probe was
+      // structurally incapable of failing, which is why celilo#1365 read as a
+      // 30s timeout and then as a green step: the step never had an opinion.
+      // curl's own %{exitcode} needs no shell variable and cannot be expanded
+      // away.
       const probe = await exec(
-        'curl -fsSL --max-time 3 https://celilo.computer/install.sh -o /dev/null 2>&1',
+        "curl -sS --max-time 3 -o /dev/null -w 'curl-exit=%{exitcode}\n' https://celilo.computer/install.sh 2>&1",
         5_000,
       );
-      if (probe.exitCode === 0) {
+      const text = `${probe.stdout}\n${probe.stderr}`.trim();
+      const code = /curl-exit=(\d+)/.exec(text);
+      // FAIL CLOSED. The probe command ends in `echo`, so the shell's exit is 0
+      // whatever curl did — falling back to probe.exitCode scored "I could not
+      // read the answer" as "ready". Measured on run 7258: the probe reported
+      // `✔ 0s` while celilo.computer was serving NO certificate at all
+      // (openssl s_client returned an empty chain and the server aborted the
+      // handshake with `internal error (592)`), and the real failure surfaced
+      // one step later as install.sh's curl (35). A probe that cannot fail is
+      // worse than no probe: it converted celilo#1365's defect into a green
+      // step plus a mystery. Unreadable now counts as not-ready, the same bias
+      // live-stack.ts already applies to its own evidence test.
+      lastProbeExit = code ? Number(code[1]) : PROBE_UNREADABLE;
+      lastProbeOutput = text;
+      if (lastProbeExit === 0) {
         caReady = true;
         break;
       }
+      // REFUSE INSTANTLY on a dead root rather than spending the 240s budget
+      // and then reporting a timeout (celilo#1427). The chain either names
+      // Pebble's live root or it does not, and that answer does not change by
+      // waiting. A cold caddy serving NOTHING yet scores `unknown` and keeps
+      // the wait, which is the case the budget exists for.
+      if (Date.now() >= staleCheckAt) {
+        staleCheckAt = Date.now() + 30_000;
+        const [live, served] = await Promise.all([
+          exec('curl -sk --max-time 5 https://100.64.0.100:15000/roots/0 | openssl x509 -noout -subject 2>/dev/null', 10_000),
+          // Deliberately a LOOSE grep: openssl's indentation for the issuer line
+          // differs across versions, and detectStaleAcmeRoot does the real
+          // parsing anyway. An anchored pattern that misses scores `unknown`,
+          // which reads exactly like a cold caddy and disables the check.
+          exec('timeout 8 openssl s_client -connect celilo.computer:443 -servername celilo.computer -showcerts </dev/null 2>/dev/null | grep i:', 12_000),
+        ]);
+        const verdict = detectStaleAcmeRoot(live.stdout, served.stdout);
+        if (verdict.kind === 'stale') {
+          console.log('✗');
+          console.error(await tlsDiagnosticLayers(exec, net.projectName));
+          throw new Error(explainStaleRoot(verdict));
+        }
+        // Settled: the served chain names the live root, so it cannot go stale
+        // by waiting. Stop paying ~20s of exec per cycle for a known answer.
+        if (verdict.kind === 'current') staleCheckAt = Number.POSITIVE_INFINITY;
+      }
       await new Promise((resolve) => setTimeout(resolve, 1000));
     }
     if (!caReady) {
-      throw new Error(
-        'Timed out (30s) waiting for celilo.computer to serve install.sh ' +
-          'over HTTPS — Pebble CA fetch likely stalled or website-sim is down.',
-      );
+      // Self-diagnosing (e2e-confidence #255, mirroring install-sh.test.ts):
+      // a bake that fails here used to name two GUESSES and discard the probe
+      // output, so celilo#1365 produced two CI failures with no cause visible.
+      // Dump the DNS / reachability / TLS layers from inside the container.
+      const layers = await tlsDiagnosticLayers(exec, net.projectName);
+      console.error(layers);
+      console.error(`--- last probe (curl exit ${lastProbeExit}) ---\n${lastProbeOutput || '(no output)'}`);
+      throw new Error(explainProbeFailure(lastProbeExit));
     }
     console.log(`✔ ${Math.round((Date.now() - tCa) / 1000)}s`);
 
@@ -340,6 +643,10 @@ async function bakeViaSim(): Promise<void> {
       console.error('--- install.sh output (tail) ---');
       console.error(result.stdout.slice(-3000));
       console.error(result.stderr.slice(-2000));
+      // The probe verified this very URL seconds ago, so a TLS failure HERE is
+      // a change between the two calls, not a static misconfiguration — and it
+      // is unreadable without the layers (run 7255).
+      console.error(await tlsDiagnosticLayers(exec, net.projectName));
       throw new Error('install.sh failed inside the bake container');
     }
     console.log(`✔ ${Math.round((Date.now() - t2) / 1000)}s`);
@@ -348,14 +655,40 @@ async function bakeViaSim(): Promise<void> {
     // CLI, genuinely against the installed binary rather than the mount.
     const version = await installShimAndVerify(exec, 'sim');
 
-    process.stdout.write('  docker commit .......... ');
-    const t5 = Date.now();
     containerId = run(
       `docker compose -f ${COMPOSE_FILE} -p ${net.projectName} ps -q management`,
     ).trim();
     if (!containerId) {
       throw new Error('Could not resolve management container id');
     }
+    // CONSUMER MODE HAS NOTHING TO VERIFY AGAINST, and the check reads a file
+    // that is deliberately not shipped. verifyInstalledCliIsStaged compares the
+    // installed CLI against the tarball restageSimulatorInputs packed, recorded
+    // in PACKAGE_ROOT/.npm-registry-cache/pack-manifest.json. Without a checkout
+    // that restage is skipped (see restageSimulatorInputs), nothing writes the
+    // manifest, and `.npm-registry-cache` is excluded from the e2e tarball on
+    // purpose (celilo#116) — so PACKAGE_ROOT resolves into node_modules and the
+    // read is a guaranteed ENOENT. sourceStamp already reasons exactly this way
+    // a few lines up.
+    //
+    // It is not merely impossible here, it is meaningless: in consumer mode the
+    // CLI came from the registry this run, so there is no staged copy for it to
+    // have drifted from. The freshness label records CONSUMER_FINGERPRINT for
+    // the same reason.
+    //
+    // Measured 2026-09-26: npm-consumer-smoke run 7335 died at 174s with
+    // `ENOENT ... node_modules/@celilo/e2e/.npm-registry-cache/pack-manifest.json`,
+    // which failed PR #1437 and blocked the release the fleet was waiting on.
+    // The check landed in 00e2367d for celilo#1423 (image-freshness certifying a
+    // stale CLI) — a real fix whose guard was written for monorepo mode only.
+    if (findMonorepoRoot(PACKAGE_ROOT)) {
+      await verifyInstalledCliIsStaged(exec, containerId);
+    } else {
+      console.log('  installed CLI = staged   skipped (consumer mode: nothing was staged to compare)');
+    }
+
+    process.stdout.write('  docker commit .......... ');
+    const t5 = Date.now();
     const superseded = imageIdOnTag('celilo-e2e/management:latest');
     // The committed container runs under the sim topology's compose env, and
     // commit captures its Env. Since the default managementZone became
@@ -377,11 +710,30 @@ async function bakeViaSim(): Promise<void> {
     );
     console.log(`✔ ${Math.round((Date.now() - t5) / 1000)}s`);
     removeSupersededImage(superseded, 'celilo-e2e/management:latest');
+  } catch (err) {
+    bakeFailed = true;
+    throw err;
   } finally {
     process.stdout.write('  tearing down ........... ');
     const tDown = Date.now();
     await net.stop();
     console.log(`✔ ${Math.round((Date.now() - tDown) / 1000)}s`);
+    if (bakeFailed) {
+      // A failed bake must not leave the shared stack live. On the builder the
+      // job dies at this step and the workflow's `if: always()` teardown steps
+      // never run (observed on runs 6645/6682/6693/6698: the log ends at
+      // "Cleaning up container for job"), so the 13-container shared stack
+      // then refuses every full run for the next 60 minutes under the
+      // orphan-age gate in live-stack.ts. Success keeps the stack: the Mac dev
+      // flow reuses it and the workflow's teardown step owns the builder's
+      // success path. (celilo#1365, celilo#1320)
+      try {
+        await stopSharedInfra();
+      } catch {
+        // best-effort: the workflow teardown or the next run's sweep is the
+        // backstop, and hiding the reason here beats failing twice.
+      }
+    }
   }
 
   console.log('');

package/config/routing/management-routes.sh

@@ -6,6 +6,17 @@ set -e
 # Generate SSH keys if needed
 /config/ssh/generate-keys.sh /ssh-keys
 
+# Serve the remote API. `celilo --remote` and celilo-web-console both arrive
+# as an ssh forced command; without sshd the console is granted in the
+# database and cannot open a connection (celilo#1442). Host keys are generated
+# per container -- clients in this rig do not pin them.
+ssh-keygen -A >/dev/null 2>&1 || true
+mkdir -p /run/sshd
+# Not fatal: this script runs under `set -e`, and letting a failed sshd abort
+# it would take the whole management box down -- turning one suite's remote-API
+# problem into every suite failing with no container.
+/usr/sbin/sshd || echo "WARNING: sshd did not start; the remote API is unreachable" >&2
+
 # Set up SSH
 mkdir -p /root/.ssh
 cp /ssh-keys/id_ed25519 /root/.ssh/id_ed25519

package/config/routing/minio-startup.sh

@@ -1,10 +1,14 @@
 #!/bin/sh
-# Start MinIO and pre-create the backup bucket.
+# Start the S3 simulator and pre-create the backup bucket.
+#
+# ⚠️ The server is s3proxy, not MinIO — see Dockerfile.minio's header for why
+# (MinIO's images stopped being anonymously pullable from both Docker Hub and
+# quay.io; celilo#1389). The hostname `minio.lab` and this file's name are
+# deliberately unchanged.
 #
 # celilo's S3 provider never creates buckets (its initialize() only probes via
 # ListObjectsV2), so the bucket must exist before `storage add s3` verifies or
-# `backup --storage` uploads. We start the server, wait for it to answer, then
-# `mc mb --ignore-existing`.
+# `backup --storage` uploads.
 set -e
 
 # Route the customer's public prefix via the ISP edge (fw-ext), like every
@@ -12,38 +16,29 @@ set -e
 # across networks, so without this a reply to the customer's WAN address is
 # dropped and `backup --storage` hangs. fw-ext is per-test, so this is
 # expected to fail while shared infra runs alone — hence the tolerance.
-# (`busybox ip` because the MinIO image ships no iproute2.)
+# (`busybox ip` because the s3proxy image ships no iproute2.)
 busybox ip route del default 2>/dev/null || true
 busybox ip route add default via 100.64.0.1 2>/dev/null || true
 
+# Pre-create the bucket. Under the jclouds filesystem backend a bucket IS a
+# directory under JCLOUDS_FILESYSTEM_BASEDIR, so this needs no running server,
+# no credentials and no client binary — which is the entire reason the
+# `minio/mc` image is gone. Measured 2026-09-24: with only this mkdir,
+# ListObjectsV2 on the bucket answers 200 and an object PUT/GET round-trips.
+#
+# It also means the bucket is ready BEFORE the server accepts a connection,
+# so the old "wait up to 30s for the admin alias, then mc mb" loop is gone.
+# There is no readiness race left to lose.
+mkdir -p "${JCLOUDS_FILESYSTEM_BASEDIR}/${MINIO_BUCKET}"
+echo "minio-startup: bucket '${MINIO_BUCKET}' ready at ${JCLOUDS_FILESYSTEM_BASEDIR}/${MINIO_BUCKET}"
+
 # Serve the S3 API on :80 so the management box reaches it over the SAME
 # proven HTTP egress path as the other public sims (apt-repo, npm-registry):
-# fw-ext's Squid-bypass exception is scoped to dport 80/443 → 100.64.0.0/24, so
+# fw-ext's Squid-bypass exception is scoped to dport 80/443 -> 100.64.0.0/24, so
 # plain HTTP to minio.lab:80 forwards straight through (no TLS bump, no cert,
-# no nonstandard-port filtering). Console stays on :9001.
-minio server /data --address ":80" --console-address ":9001" &
-MINIO_PID=$!
-
-# Wait for the server to accept the admin alias (up to ~30s).
-ready=0
-i=0
-while [ "$i" -lt 30 ]; do
-  if mc alias set local "http://127.0.0.1:80" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; then
-    ready=1
-    break
-  fi
-  i=$((i + 1))
-  sleep 1
-done
-
-if [ "$ready" -ne 1 ]; then
-  echo "minio-startup: MinIO did not become ready in 30s" >&2
-  exit 1
-fi
-
-mc mb --ignore-existing "local/${MINIO_BUCKET}"
-echo "minio-startup: bucket '${MINIO_BUCKET}' ready; MinIO serving on :80"
-
-# Hand the foreground back to the server so the container stays up and signals
-# propagate to it.
-wait "$MINIO_PID"
+# no nonstandard-port filtering). S3PROXY_ENDPOINT pins the listen address.
+#
+# exec so the server becomes PID 1's child directly and signals propagate; the
+# old script backgrounded the server and `wait`ed on it, which was only needed
+# because the bucket step ran after startup.
+exec /opt/s3proxy/run-docker-container.sh

package/config/routing/resolver-internal-routes.sh

@@ -26,12 +26,19 @@ ip route add default via "$GATEWAY"
 SPLIT_HORIZON_BLOCK=""
 IFS=',' read -ra DOMAIN_LIST <<< "${DOMAINS}"
 for d in "${DOMAIN_LIST[@]}"; do
-    SPLIT_HORIZON_BLOCK="${SPLIT_HORIZON_BLOCK}    local-zone: \"${d}.\" redirect\n    local-data: \"${d}. IN A ${CADDY_IP}\"\n"
+    SPLIT_HORIZON_BLOCK="${SPLIT_HORIZON_BLOCK}    local-zone: \"${d}.\" redirect
+    local-data: \"${d}. IN A ${CADDY_IP}\"
+"
 done
 
 # Generate unbound config with split-horizon for each test domain plus
 # the standard simulator overrides (Pebble, registry).
-cat > /etc/unbound/unbound.conf << EOF
+# Quoted heredocs: nothing inside is expanded, so a backtick or a $ in a
+# comment is text, not code. The split-horizon block is printed BETWEEN them
+# as data. (A `internal` in a comment used to run as a command on every boot —
+# celilo#1430.)
+{
+cat << 'EOF'
 server:
     interface: 0.0.0.0
     access-control: 0.0.0.0/0 allow
@@ -41,7 +48,9 @@ server:
     do-daemonize: no
     do-not-query-localhost: no
 
-$(printf "${SPLIT_HORIZON_BLOCK}")
+EOF
+printf '%s' "${SPLIT_HORIZON_BLOCK}"
+cat << 'EOF'
     # Mirror the public resolver's simulated endpoint overrides so that
     # management uses Pebble for ACME and the e2e registry for modules.
     local-data: "acme-v02.api.letsencrypt.org. IN A 100.64.0.100"
@@ -52,7 +61,6 @@ $(printf "${SPLIT_HORIZON_BLOCK}")
     local-data: "npm-registry.lab. IN A 100.64.0.59"
     # apt-repo simulator — serves the celilo + celilo-bootstrap .debs to the
     # apt install step in the bootstrap-apt test. See SIMULATOR_IPS.APT_REPO.
-    # (No backticks in this heredoc — it is unquoted, so backticks execute.)
     local-data: "apt.celilo.lab. IN A 100.64.0.60"
     # MinIO S3 simulator — backup/restore target for the migration e2e.
     # See SIMULATOR_IPS.MINIO.
@@ -60,10 +68,9 @@ $(printf "${SPLIT_HORIZON_BLOCK}")
     # IP echo simulator — the rig's api.ipify.org. celilo's public_dns check
     # asks it the one thing the fleet cannot ask itself: what address it is
     # currently reachable at. See SIMULATOR_IPS.IP_ECHO.
-    # (Backtick-free on purpose — see the note above; this heredoc is unquoted.)
     local-data: "ip-echo.lab. IN A 100.64.0.65"
     # signal-cli daemon (real, unlinked) and its simulator counterpart. Both
-    # sit on the `internal` network — celilo-mgr reaches a notification
+    # sit on the internal network — celilo-mgr reaches a notification
     # transport locally; a transport needing public ingress could not tell
     # you the ingress was broken.
     local-data: "signal-cli.lab. IN A 10.226.1.90"
@@ -77,6 +84,7 @@ forward-zone:
     name: "."
     forward-addr: 203.0.113.1
 EOF
+} > /etc/unbound/unbound.conf
 
 /usr/sbin/unbound -d &
 

package/config/socks/startup.sh

@@ -47,4 +47,50 @@ echo "nameserver $NAMESERVER" > /etc/resolv.conf
 
 echo "socks-proxy ready: vantage=$VANTAGE gateway=$GATEWAY dns=$NAMESERVER" >&2
 
+# TEACH DANTE ABOUT THE TUNNEL ONCE IT EXISTS.
+#
+# danted.conf names `external: eth0`, the interface dante binds OUTBOUND
+# connections to. That is correct at startup and wrong the moment a test brings
+# up a WireGuard tunnel: a destination routed over wg0 is then dialled from
+# eth0's address, and the far side drops it. The symptom is that a client INSIDE
+# this container reaches the target fine (ordinary routing picks wg0) while the
+# same target through the SOCKS port times out — so the proxy looks up, the
+# tunnel looks up, and only the relay fails.
+#
+# Measured 2026-09-26 (celilo#1438): celilo-web-console-oidc stage 6 passed
+# (its curl runs inside the container) and stage 7 failed with
+# `net::ERR_TIMED_OUT` on the identical URL through the proxy. From the host:
+# inside=302, via-socks=000.
+#
+# wg0 does not exist when danted starts, so it cannot simply be listed. This
+# watches for it, adds it, and reloads — and VERIFIES the config first, because
+# a danted that dies on a bad reload takes the proxy out for every suite.
+(
+  while :; do
+    if ip link show wg0 >/dev/null 2>&1; then
+      if ! grep -q '^external: wg0' /etc/danted.conf; then
+        cp /etc/danted.conf /tmp/danted.conf.pre-wg0
+        sed -i 's|^external: eth0$|external: eth0\nexternal: wg0\nexternal.rotation: route|' /etc/danted.conf
+        # READ THE OUTPUT, NOT THE EXIT CODE. `danted -V` exits 0 even on a
+        # config it refuses outright — measured 2026-09-26: a config naming a
+        # nonexistent interface logged `error: ... could not resolve hostname`
+        # and `alert: mother[1/1]: shutting down`, and still returned 0. A guard
+        # keyed on the status would have HUPped a config danted cannot load and
+        # killed the proxy for every suite, which is the thing it exists to stop.
+        /usr/sbin/danted -V -f /etc/danted.conf >/tmp/danted-verify.log 2>&1 || true
+        if ! grep -qiE '(^|[^a-z])(error|alert):' /tmp/danted-verify.log; then
+          kill -HUP 1 2>/dev/null || true
+          echo "socks-proxy: wg0 up, danted now routes outbound by route" >&2
+        else
+          cp /tmp/danted.conf.pre-wg0 /etc/danted.conf
+          echo "socks-proxy: REFUSED to reload danted, config invalid — proxy left on eth0 only:" >&2
+          tail -3 /tmp/danted-verify.log >&2
+        fi
+      fi
+      break
+    fi
+    sleep 1
+  done
+) &
+
 exec /usr/sbin/danted -f /etc/danted.conf