@celilo/e2e 0.20.6 → 0.21.1

package/src/installed-cli-digest.ts

@@ -0,0 +1,102 @@
+/**
+ * Prove the CLI install.sh actually installed is the CLI this bake packed.
+ *
+ * `image-freshness` stamps the fingerprint of the STAGED tarballs
+ * (pack-manifest.json). That is an attestation of intent: it says what the bake
+ * meant to install, never what it installed. The two diverge whenever the
+ * tarballs reach the container by a path that can go stale — the npm-registry
+ * sim COPYs `.npm-registry-cache/` at docker-BUILD time, so a sim image that
+ * failed to rebuild keeps serving last week's tarball while the manifest beside
+ * it describes today's tree (celilo#1423, and the staging half in celilo#1299).
+ *
+ * Measured 2026-09-25: doctor reported `baked from this tree (2496fd8eaf07f66f)`
+ * over an image whose installed `@celilo/cli` was the pre-fix revision. A
+ * 16-minute run on a P1 was then read as falsifying a correct hypothesis — a fix
+ * that did not ship looks exactly like a fix that did not work, and this gate is
+ * the one instrument whose job is telling those apart.
+ *
+ * So: compare bytes. Copy the staged tarball into the container, digest the
+ * files it contains from BOTH the extracted tarball and the installed package,
+ * and fail the bake when they differ — before `docker commit` can stamp a label
+ * that would outlive the mistake. Fail-closed at bake time beats a warning at
+ * doctor time: the wrong image never gets committed.
+ *
+ * Only paths present in the TARBALL are digested. An installer is free to add
+ * files (bun does not, today) and a test that forbade it would fail for a reason
+ * nobody cares about; a tarball file that is MISSING from the install still
+ * fails, because sha256sum errors on it.
+ */
+
+/** Where install.sh leaves the global @celilo/cli inside the management image. */
+export const INSTALLED_CLI_DIR = '/root/.bun/install/global/node_modules/@celilo/cli';
+
+/** Container paths the probe uses for its own scratch. */
+const STAGED_TARBALL = '/tmp/cli-staged.tgz';
+const STAGED_DIR = '/tmp/cli-staged';
+
+/**
+ * The shell script that prints both digests.
+ *
+ * Returned as text for the caller to base64 into the container. It must NOT be
+ * interpolated into a dockerExec command directly: dockerExec wraps the command
+ * as `bash -c "<JSON>"` and hands it to a HOST shell, which expands `$(...)`
+ * and `$VAR` itself before the container ever sees them (celilo#1365 lost days
+ * to exactly that, via `$?`). base64 has no metacharacters.
+ */
+export function digestScript(installedDir: string = INSTALLED_CLI_DIR): string {
+  return [
+    '#!/bin/bash',
+    'set -euo pipefail',
+    `rm -rf ${STAGED_DIR}`,
+    `mkdir -p ${STAGED_DIR}`,
+    `tar -xzf ${STAGED_TARBALL} -C ${STAGED_DIR}`,
+    // npm/bun tarballs root everything under `package/`.
+    `S=${STAGED_DIR}/package`,
+    `I=${installedDir}`,
+    // One list, taken from the tarball, digested against both trees — so the
+    // comparison is over the same paths in the same order on both sides.
+    // sha256sum on the container, shasum on a macOS dev box. Resolved once so
+    // the gate below is runnable everywhere — a test that only runs in CI is a
+    // test nobody has watched fail.
+    'if command -v sha256sum >/dev/null; then SUM="sha256sum"; else SUM="shasum -a 256"; fi',
+    'cd "$S" && find . -type f -print | LC_ALL=C sort > /tmp/cli-files',
+    // `while read` rather than xargs: portable, and it keeps one file per line
+    // so a missing file still errors the pipeline under `set -o pipefail`.
+    'digest() { (cd "$1" && while IFS= read -r f; do $SUM "$f"; done < /tmp/cli-files) | $SUM | cut -d" " -f1; }',
+    'echo "staged=$(digest "$S")"',
+    'echo "installed=$(digest "$I")"',
+  ].join('\n');
+}
+
+export interface DigestPair {
+  staged: string;
+  installed: string;
+}
+
+/**
+ * Pull the two digests out of the probe's output.
+ *
+ * Throws rather than returning a partial: an unreadable probe is the
+ * can't-fail-shaped failure this whole file exists to prevent, so it must not
+ * degrade into a pass.
+ */
+export function parseDigests(output: string): DigestPair {
+  const read = (key: string): string => {
+    const match = output.match(new RegExp(`^${key}=([0-9a-f]{64})$`, 'm'));
+    if (!match) {
+      throw new Error(
+        `installed-CLI probe printed no ${key} digest — it did not run to completion.\n` +
+          `--- probe output ---\n${output.trim() || '(empty)'}`,
+      );
+    }
+    return match[1] as string;
+  };
+  return { staged: read('staged'), installed: read('installed') };
+}
+
+/** The bake-stopping message for a mismatch, naming the cause worth checking first. */
+export function mismatchMessage({ staged, installed }: DigestPair): string {
+  return `The installed @celilo/cli is NOT the tarball this bake staged.\n  staged tarball: ${staged}\n  installed CLI:  ${installed}\ninstall.sh fetched a DIFFERENT build from the npm-registry sim. The sim image COPYs .npm-registry-cache/ at docker-build time, so re-staging tarballs does nothing until that image is rebuilt — and build.ts scans image directories in sorted order and exits on the first failure, so an image failing earlier in the alphabet (minio, say) strands the registry sim while a usable-looking :latest remains. Rebuild the sims: \`cele2e down && cele2e build-infra\`.\nRefusing to commit — a stamped label over the wrong CLI is celilo#1423.`;
+}
+
+export { STAGED_TARBALL, STAGED_DIR };

package/src/parse-line.ts

@@ -166,7 +166,13 @@ export function parseLine(
   // for failing tests.
   const errorMatch = line.match(/\b(?:[A-Z]\w*)?Error: (.*)/);
   if (errorMatch && !line.includes('bun test') && !line.includes('ipv4: Address already')) {
-    const err = errorMatch[1].slice(0, 70);
-    display.subEvent(`${red}✗${reset} ${err}`);
+    // Pass the whole message. `subEvent` already fits it to the terminal
+    // and appends a visible `…` when it clips, so a second fixed-width
+    // slice here only made the clip SILENT and narrower than the screen.
+    // Measured 2026-09-25: a 6b stage-1 failure rendered as
+    // "No infrastructure " — exactly 70 chars, cutting immediately before
+    // the zone name, which is the one fact that localises the defect. It
+    // read as a complete sentence and cost a wrong diagnosis.
+    display.subEvent(`${red}✗${reset} ${errorMatch[1]}`);
   }
 }

package/src/resolver-internal-config.test.ts

@@ -0,0 +1,65 @@
+/**
+ * Recurrence gate for celilo#1430: dns-int logged `internal: command not found`
+ * on every boot.
+ *
+ * The unbound config was written through an UNQUOTED heredoc, so a backtick in
+ * a comment ran as a command and the split-horizon block was passed as
+ * printf's FORMAT STRING. Both are silent: unbound still started, so the only
+ * symptom was noise in the middle of the resolver's readiness path.
+ *
+ * The check runs the real script's config-rendering half in bash and asserts
+ * nothing reaches stderr — the one signal a shell-injected comment produces.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+const SCRIPT = join(import.meta.dir, '../config/routing/resolver-internal-routes.sh');
+
+/** Run the script with `ip`/`unbound` stubbed and the config written to stdout. */
+function render(domains: string): { config: string; stderr: string } {
+  const harness = `
+    PATH="$PWD/.stub:$PATH"
+    mkdir -p .stub
+    printf '#!/bin/sh\\nexit 0\\n' > .stub/ip
+    printf '#!/bin/sh\\nexit 0\\n' > .stub/unbound
+    printf '#!/bin/sh\\nexit 0\\n' > .stub/sleep
+    chmod +x .stub/ip .stub/unbound .stub/sleep
+    mkdir -p etc/unbound
+    sed -e 's#/etc/unbound/unbound.conf#etc/unbound/unbound.conf#' \
+        -e 's#/usr/sbin/unbound#unbound#' ${JSON.stringify(SCRIPT)} > run.sh
+    bash run.sh > /dev/null
+    cat etc/unbound/unbound.conf
+  `;
+  const r = spawnSync('bash', ['-c', harness], {
+    cwd: mkdtempSync(join(tmpdir(), 'celilo-1430-')),
+    encoding: 'utf8',
+    env: { ...process.env, DOMAINS: domains },
+  });
+  return { config: r.stdout ?? '', stderr: r.stderr ?? '' };
+}
+
+describe('resolver-internal-routes.sh', () => {
+  test('renders a multi-domain config with nothing on stderr', () => {
+    const { config, stderr } = render('iamtheinternet.org,example.net');
+    expect(stderr).toBe('');
+    expect(config).toContain('local-zone: "iamtheinternet.org." redirect');
+    expect(config).toContain('local-zone: "example.net." redirect');
+    expect(config).not.toContain('\\n');
+  });
+
+  test('a single domain renders the same shape', () => {
+    const { config, stderr } = render('iamtheinternet.org');
+    expect(stderr).toBe('');
+    expect(config).toContain('local-zone: "iamtheinternet.org." redirect');
+    expect(config).not.toContain('example.net');
+  });
+
+  test('the config heredocs are quoted, so a comment can never run', () => {
+    const src = readFileSync(SCRIPT, 'utf8');
+    expect(src).not.toMatch(/<<\s*EOF/);
+  });
+});

package/src/shared-infra-pebble-root.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, test } from 'bun:test';
+import { pebbleMintedANewRoot } from './shared-infra';
+
+/**
+ * Pebble mints a random root CA at every startup and Caddy caches the cert it
+ * was issued, so a Pebble that restarts without its dependants restarting
+ * leaves them serving a cert from a root that no longer exists (celilo#1427).
+ * Recreating the sims is how that is avoided — but recreating them when Pebble
+ * did NOT restart costs a rebuild on every run, so the predicate has to be
+ * exact in both directions.
+ */
+describe('pebbleMintedANewRoot', () => {
+  test('a different container id means a new root', () => {
+    expect(pebbleMintedANewRoot('abc123', 'def456')).toBe(true);
+  });
+
+  test('the same container id means the root is unchanged', () => {
+    expect(pebbleMintedANewRoot('abc123', 'abc123')).toBe(false);
+  });
+
+  test('an unreadable id is not evidence of a restart', () => {
+    expect(pebbleMintedANewRoot('', 'def456')).toBe(false);
+    expect(pebbleMintedANewRoot('abc123', '')).toBe(false);
+    expect(pebbleMintedANewRoot('', '')).toBe(false);
+  });
+});

package/src/shared-infra-self-contained.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from 'bun:test';
+import { generateSharedInfraYaml } from './docker-compose-generator';
+
+/**
+ * Shared infrastructure must be self-contained.
+ *
+ * Shared infra starts BEFORE any per-test stack exists, so a shared service
+ * that names a per-test address gets no answer at all until a test comes up.
+ * That is celilo#1365: the celilo.computer simulator resolved ACME through
+ * comcast-resolver (203.0.113.1, per-test) and Pebble validated challenges
+ * through the same address, so Caddy could not obtain a cert during
+ * shared-infra bringup. The bake then probed https://celilo.computer within
+ * 30s, got whatever Caddy serves without an ACME cert, and failed TLS
+ * verification — "unable to get local issuer certificate", which reads as a
+ * CA-trust bug rather than a startup-order one.
+ *
+ * The check is derived, not hand-listed: every address shared infra POINTS AT
+ * for name resolution must belong to a service shared infra itself defines.
+ */
+describe('shared infra is self-contained', () => {
+  const yaml = generateSharedInfraYaml();
+  const compose = Bun.YAML.parse(yaml) as {
+    services: Record<
+      string,
+      { dns?: string[]; command?: string; networks?: Record<string, { ipv4_address?: string }> }
+    >;
+  };
+
+  const ownAddresses = new Set(
+    Object.values(compose.services).flatMap((svc) =>
+      Object.values(svc.networks ?? {})
+        .map((n) => n.ipv4_address)
+        .filter((ip): ip is string => Boolean(ip)),
+    ),
+  );
+
+  // Every resolver a shared service is pointed at, as {who, address}.
+  const resolverRefs = Object.entries(compose.services).flatMap(([name, svc]) => {
+    const refs = (svc.dns ?? []).map((ip) => ({ name, field: 'dns', ip }));
+    const dnsserver = /-dnsserver\s+(\d+\.\d+\.\d+\.\d+):/.exec(svc.command ?? '');
+    if (dnsserver) refs.push({ name, field: '-dnsserver', ip: dnsserver[1] });
+    return refs;
+  });
+
+  test('the generator actually emits resolver references to check', () => {
+    expect(resolverRefs.length).toBeGreaterThan(0);
+  });
+
+  test('no shared service resolves through an address shared infra does not own', () => {
+    const foreign = resolverRefs.filter((ref) => !ownAddresses.has(ref.ip));
+    expect(foreign.map((r) => `${r.name}.${r.field} -> ${r.ip}`)).toEqual([]);
+  });
+});

package/src/shared-infra.ts

@@ -161,6 +161,65 @@ function getE2eDir(): string {
  * Start the shared infrastructure. Idempotent — if already running,
  * verifies health and returns immediately.
  */
+/**
+ * Services whose Caddy obtains an ACME cert from Pebble and then CACHES it
+ * until expiry.
+ */
+const PEBBLE_DEPENDENT_SIMS = ['celilo-website', 'isitup'];
+
+function containerIdOf(e2eDir: string, service: string): string {
+  try {
+    return run(
+      `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} ps -q ${service}`,
+      {
+        cwd: e2eDir,
+        timeout: 15_000,
+      },
+    ).trim();
+  } catch {
+    return '';
+  }
+}
+
+/**
+ * Pebble mints a RANDOM root CA at every startup. Caddy caches the cert it was
+ * issued and will not re-request until expiry, so a Pebble that restarts
+ * WITHOUT its dependants restarting leaves them serving a cert signed by a root
+ * that no longer exists. Every probe that waits for HTTPS to VERIFY then waits
+ * out its whole budget, which is why celilo#1365 and celilo#1427 both presented
+ * as timeouts: `curl -k` returns 200 the entire time.
+ *
+ * `docker compose up -d` recreates only what changed, so this is the ordinary
+ * case, not an edge one — a rebuilt Pebble image against an unchanged website
+ * sim produces it every time.
+ *
+ * Restarting the sim alone does not fix it: it comes back on Docker's embedded
+ * resolver and cannot reach the simulated ACME endpoint at all (measured,
+ * crew-builder, celilo#1427). Recreating it through compose keeps its
+ * configured `dns:`, so it re-runs ACME against the live root.
+ */
+export function pebbleMintedANewRoot(idBefore: string, idAfter: string): boolean {
+  // An empty id on either side means "we could not tell" — Pebble was absent
+  // before (first start: the sims come up fresh anyway) or could not be read
+  // after. Neither is evidence of a restart, and recreating on no evidence
+  // would rebuild the sims on every single run.
+  if (!idBefore || !idAfter) return false;
+  return idBefore !== idAfter;
+}
+
+function recreateSimsWhosePebbleRestarted(e2eDir: string, pebbleIdBefore: string): void {
+  const pebbleIdAfter = containerIdOf(e2eDir, 'letsencrypt');
+  if (!pebbleMintedANewRoot(pebbleIdBefore, pebbleIdAfter)) return;
+
+  console.log(
+    '[progress:sub] Pebble was recreated (new root CA) — recreating the sims that cache its certs',
+  );
+  run(
+    `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d --force-recreate ${PEBBLE_DEPENDENT_SIMS.join(' ')}`,
+    { cwd: e2eDir, timeout: 120_000 },
+  );
+}
+
 export async function ensureSharedInfra(): Promise<void> {
   if (isSharedInfraRunning()) {
     // Verify DNS is still healthy
@@ -247,10 +306,12 @@ export async function ensureSharedInfra(): Promise<void> {
   // baked tag fails right here, naming the image. startNetwork checks the
   // full referenced set first and names the remedy (cele2e build-infra).
   console.log('[progress:start] starting shared infrastructure | shared infra started');
+  const pebbleIdBefore = containerIdOf(e2eDir, 'letsencrypt');
   run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d`, {
     cwd: e2eDir,
     timeout: 120_000,
   });
+  recreateSimsWhosePebbleRestarted(e2eDir, pebbleIdBefore);
 
   // Wait for DNS convergence — robustly. The old single 60s window with no
   // recovery was an intermittent build-infra failure on the builder (#319:

package/src/socks-proxy.ts

@@ -238,11 +238,24 @@ function makeHandle(containerName: string, port: number): SocksProxyHandle {
         return { stdout, stderr: '', exitCode: 0 };
       } catch (err: unknown) {
         const e = err as { stdout?: Buffer; stderr?: Buffer; status?: number };
-        return {
-          stdout: e.stdout?.toString() ?? '',
-          stderr: e.stderr?.toString() ?? String(err),
-          exitCode: e.status ?? 1,
-        };
+        const stdout = e.stdout?.toString() ?? '';
+        const stderr = e.stderr?.toString() ?? String(err);
+        // A MISSING BINARY MUST NOT READ AS AN EMPTY ANSWER. Callers assert on
+        // stdout — `expect(probe.stdout.trim()).toBe('302')` — so a command
+        // that never ran returns '' and is indistinguishable from a server that
+        // answered nothing. celilo#1426's stage 6 failed exactly that way: the
+        // proxy image had no curl, `Expected "302", Received ""`, and the
+        // investigation went to the tunnel, the firewall and DNS before the
+        // container. 127 is the shell's not-found status; surface it in stdout
+        // so the assertion's own diff names the cause.
+        if (e.status === 127) {
+          return {
+            stdout: `<command not found in the proxy container: ${stderr.trim() || cmd}>`,
+            stderr,
+            exitCode: 127,
+          };
+        }
+        return { stdout, stderr, exitCode: e.status ?? 1 };
       }
     },
     async stop(): Promise<void> {

package/src/stale-acme-root.ts

@@ -0,0 +1,66 @@
+/**
+ * Tell a STALE Pebble root apart from an unfinished ACME issuance.
+ *
+ * Pebble mints a random root CA at every startup and Caddy caches its
+ * certificate until expiry, so a Pebble that restarted without the website
+ * sim restarting serves a cert whose issuing root no longer exists. The bake
+ * probe waits for HTTPS to VERIFY, so that trust failure spends the whole
+ * 240s budget and then reports a TIMEOUT — and waiting longer cannot fix it
+ * (celilo#1427).
+ *
+ * The two conditions need opposite responses and curl's exit code cannot
+ * separate them: a cold caddy that has obtained nothing yet also fails the
+ * handshake. The evidence that CAN is the chain itself — compare the root CN
+ * Pebble is serving right now against the issuers in the chain the website
+ * hands out.
+ *
+ * Both inputs are openssl output captured inside the management container:
+ *   live: `curl -sk https://100.64.0.100:15000/roots/0 | openssl x509 -noout -subject`
+ *   served: `openssl s_client -showcerts ... | grep '^   i:'`
+ */
+
+export type StaleRootVerdict =
+  /** The served chain is signed by the root Pebble is serving now. */
+  | { kind: 'current' }
+  /** Not enough evidence — no chain served yet, or no live root readable. */
+  | { kind: 'unknown'; reason: string }
+  /** The served chain names a root Pebble no longer has. Fatal, instantly. */
+  | { kind: 'stale'; liveRoot: string; servedIssuers: string[] };
+
+/** `subject=CN = Pebble Root CA 2a43b9` / `subject= CN=Pebble Root CA 2a43b9` -> the CN. */
+function commonName(line: string): string {
+  const m = /\bCN\s*=\s*([^,/\n]+)/.exec(line);
+  return m ? m[1].trim() : '';
+}
+
+export function detectStaleAcmeRoot(
+  liveRootSubject: string,
+  servedChain: string,
+): StaleRootVerdict {
+  const liveRoot = commonName(liveRootSubject);
+  if (!liveRoot) {
+    return { kind: 'unknown', reason: 'no live Pebble root CN readable' };
+  }
+  const servedIssuers = servedChain
+    .split('\n')
+    .filter((l) => /(^|\s)i[:=]/.test(l))
+    .map(commonName)
+    .filter(Boolean);
+  if (servedIssuers.length === 0) {
+    return { kind: 'unknown', reason: 'no certificate chain served yet' };
+  }
+  if (servedIssuers.includes(liveRoot)) {
+    return { kind: 'current' };
+  }
+  return { kind: 'stale', liveRoot, servedIssuers };
+}
+
+export function explainStaleRoot(v: Extract<StaleRootVerdict, { kind: 'stale' }>): string {
+  return [
+    `celilo.computer is serving a certificate from a DEAD Pebble root. Chain issuers: ${v.servedIssuers.join(' <- ')}. Pebble's live root is "${v.liveRoot}".`,
+    'Pebble mints a random root at every startup and Caddy caches its cert until expiry, so Pebble restarted without the website sim restarting (celilo#1427).',
+    'This is NOT a timeout and waiting cannot fix it.',
+    'Remedy: `cele2e down` then `cele2e build-infra`, so Pebble and Caddy start together.',
+    'Restarting the website sim ALONE makes it worse — it comes back on Docker embedded DNS and cannot reach the simulated ACME endpoint at all.',
+  ].join(' ');
+}