@celilo/cli 1.14.0 → 2.0.0

package/src/hooks/remote-broker.ts

@@ -0,0 +1,404 @@
+/**
+ * The remote-ops broker — stage 3's answering half (hook-process-boundary,
+ * design D12).
+ *
+ * A hook holds no SSH credential: the jail does not bind `~/.ssh`, so a
+ * hand-built `ssh` cannot authenticate anywhere. `@celilo/capabilities`'
+ * remote primitives detect the hook environment and send each operation here
+ * as a STRUCTURED request. This side holds the key, checks the target against
+ * the policy the caller injected, and runs the real primitive.
+ *
+ * Structured, never a shell string: a broker that accepted a command line
+ * from the jail would run arbitrary code as celilo OUTSIDE the jail, which is
+ * the exact thing the boundary exists to prevent. The same reasoning refuses
+ * `opts.env` (it never crosses; the asking half says so loudly) and confines
+ * the stream primitives' LOCAL paths to the directories this run already owns
+ * — without that check `streamBackup` would be a write-anything-as-celilo
+ * oracle aimed at `master.key`.
+ *
+ * **Attribution (D12).** Every operation on this channel belongs to the
+ * module whose hook is running — the policy is CONSTRUCTED for that module by
+ * the invoking service. A capability provider's internal transport (the
+ * `public_web` upload) does not pass through here at all: providers run in
+ * celilo's own process (design D10), and the one hand-built-ssh provider call
+ * site is being replaced by an Ansible converge under
+ * `openspec/changes/capability-owned-tables` (celilo#1014). The same
+ * `checkTarget` contract serves a provider-owned policy if provider transport
+ * is ever brokered — the owner is a construction argument, never inferred
+ * from a caller.
+ *
+ * Unlike the capability socket (one connection, one hook), this socket
+ * answers MANY short-lived connections: the asking half spawns a client per
+ * call, one request line in, one response line out.
+ *
+ * **A remote operation blocks celilo's event loop for its duration** — the
+ * primitives are execSync-backed, bounded by each call's own `timeoutMs`.
+ * That is not new exposure: before the process boundary every hook ran its
+ * ssh on this same loop, and provider transport (design D10) still does.
+ * The asking side is blocked in spawnSync anyway, so nothing concurrent is
+ * lost; timers (the hook's total/idle kill) fire late by at most one op.
+ *
+ * Execution function (Rule 10.1) — owns the socket and performs the calls.
+ */
+
+import { lstatSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { type Server, createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { basename, dirname, join, resolve, sep } from 'node:path';
+import {
+  type RemoteTarget,
+  type RunResult,
+  type Runner,
+  execRunner,
+  installAuthorizedKey,
+  remoteExec,
+  streamBackup,
+  streamRestore,
+} from '@celilo/capabilities';
+import { z } from 'zod';
+import { createLineReader } from './hook-protocol';
+import type { HookLogger } from './types';
+
+/** The socket's filename beside the capability socket ('s') in the run dir. */
+const REMOTE_SOCKET_NAME = 'r';
+
+/** Refuse a request line larger than this rather than buffering it. */
+const MAX_REQUEST_BYTES = 64 * 1024 * 1024;
+
+const TargetSchema = z.object({
+  ipv4_address: z.string().min(1),
+  user: z.string().optional(),
+  port: z.number().int().optional(),
+  /**
+   * The module's OWN key, as content (task 5.4). The asking half reads the
+   * file inside the jail, so a hook can only send bytes it may already read
+   * — never a path for celilo to open on its behalf.
+   */
+  identityContent: z.string().optional(),
+});
+
+const OptsSchema = z
+  .object({
+    timeoutMs: z.number().optional(),
+    input: z.string().optional(),
+  })
+  .optional();
+
+const RemoteRequestSchema = z.discriminatedUnion('op', [
+  z.object({
+    op: z.literal('remoteExec'),
+    target: TargetSchema,
+    command: z.string(),
+    opts: OptsSchema,
+  }),
+  z.object({
+    op: z.literal('streamBackup'),
+    target: TargetSchema,
+    producerCommand: z.string(),
+    localPath: z.string(),
+    opts: OptsSchema,
+  }),
+  z.object({
+    op: z.literal('streamRestore'),
+    target: TargetSchema,
+    localPath: z.string(),
+    consumerCommand: z.string(),
+    opts: OptsSchema,
+  }),
+  z.object({
+    op: z.literal('installAuthorizedKey'),
+    target: TargetSchema,
+    password: z.string(),
+    opts: OptsSchema,
+  }),
+  z.object({ op: z.literal('checkTarget'), target: TargetSchema }),
+]);
+
+type RemoteRequest = z.infer<typeof RemoteRequestSchema>;
+type BridgeTarget = z.infer<typeof TargetSchema>;
+
+export type RemoteCheckResult = { allowed: true } | { allowed: false; message: string };
+
+/**
+ * D12's reachability policy, built by the invoking service for the module the
+ * run belongs to (`services/remote-access.ts`). The broker only enforces it.
+ */
+export interface RemoteAccessPolicy {
+  /** The module every operation on this channel is attributed to. */
+  moduleId: string;
+  /**
+   * May this module reach `target`? Called per operation, so it reads fresh
+   * state. `hasOwnCredential` is true when the request carries the module's
+   * own key or password rather than riding celilo's fleet key.
+   */
+  checkTarget(
+    target: { ipv4_address: string; user?: string },
+    hasOwnCredential: boolean,
+  ): RemoteCheckResult;
+}
+
+export interface RemoteBrokerOptions {
+  /** Directory holding the capability socket; the remote socket sits beside it. */
+  socketDir: string;
+  /**
+   * Absent means DENY: a run started without a policy refuses every
+   * operation, naming the gap. Fail closed, never open (Rule 6.4).
+   */
+  policy?: RemoteAccessPolicy;
+  /** Local roots `streamRestore` may read from. Realpath'd here once. */
+  readableRoots: readonly string[];
+  /** Local roots `streamBackup` may write into. Realpath'd here once. */
+  writableRoots: readonly string[];
+  logger: HookLogger;
+  /** Every operation feeds the idle tracker — remote work is real work. */
+  onActivity: () => void;
+  /** Injectable transport for tests (default: the real execRunner). */
+  runner?: Runner;
+}
+
+export interface RemoteBroker {
+  /** Unix socket path, handed to the child as CELILO_HOOK_REMOTE_SOCKET. */
+  socketPath: string;
+  /** Refuse further operations. Idempotent. Called on kill and on cleanup. */
+  stop(): void;
+  /** Close the listener. The socket file goes with the run directory. */
+  close(): void;
+}
+
+/** Serve remote operations for one hook run. */
+export async function startRemoteBroker(options: RemoteBrokerOptions): Promise<RemoteBroker> {
+  const socketPath = join(options.socketDir, REMOTE_SOCKET_NAME);
+  const runner = options.runner ?? execRunner;
+  const readableRoots = options.readableRoots.map(realpathOrSelf);
+  const writableRoots = options.writableRoots.map(realpathOrSelf);
+  let stopped = false;
+
+  const server: Server = createServer((socket) => {
+    socket.setEncoding('utf-8');
+    let answered = false;
+    let received = 0;
+
+    const answer = (response: object) => {
+      if (answered) return;
+      answered = true;
+      socket.end(`${JSON.stringify(response)}\n`);
+    };
+
+    const feed = createLineReader((line) => {
+      if (answered) return;
+      options.onActivity();
+
+      let parsed: RemoteRequest;
+      try {
+        parsed = RemoteRequestSchema.parse(JSON.parse(line));
+      } catch (error) {
+        answer(
+          failure(
+            `remote-ops broker: malformed request (${error instanceof Error ? error.message : String(error)})`,
+          ),
+        );
+        return;
+      }
+
+      if (stopped) {
+        // Same rule as the capability broker: the kill is only real if the
+        // channel refuses afterwards (celilo#1003).
+        answer(refusal(parsed.op, 'Hook run has ended; refusing remote operation.'));
+        return;
+      }
+
+      answer(dispatch(parsed));
+    });
+
+    socket.on('data', (chunk: string) => {
+      received += chunk.length;
+      if (received > MAX_REQUEST_BYTES) {
+        answer(failure('remote-ops broker: request too large'));
+        socket.destroy();
+        return;
+      }
+      feed(chunk);
+    });
+    socket.on('error', () => {
+      // A client that died mid-request has nobody to answer.
+    });
+  });
+
+  function dispatch(request: RemoteRequest): object {
+    const hasOwnCredential =
+      request.op === 'installAuthorizedKey' || request.target.identityContent !== undefined;
+
+    const verdict: RemoteCheckResult = options.policy
+      ? options.policy.checkTarget(
+          { ipv4_address: request.target.ipv4_address, user: request.target.user },
+          hasOwnCredential,
+        )
+      : {
+          allowed: false,
+          message:
+            'This hook invocation was started without a remote-access policy, so remote operations are unavailable. This is a celilo defect — the invoking service must pass one.',
+        };
+
+    if (!verdict.allowed) {
+      options.logger.warn(`remote-ops refused: ${verdict.message}`);
+      return refusal(request.op, verdict.message);
+    }
+
+    try {
+      return { ...perform(request) };
+    } catch (error) {
+      return failure(
+        `remote-ops broker: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  function perform(request: RemoteRequest): RunResult | RemoteCheckResult {
+    switch (request.op) {
+      case 'checkTarget':
+        return { allowed: true };
+
+      case 'remoteExec':
+        return withIdentity(request.target, (target) =>
+          remoteExec(target, request.command, request.opts ?? {}, runner),
+        );
+
+      case 'streamBackup': {
+        const local = containedPath(request.localPath, writableRoots, 'write');
+        if ('error' in local) return { ok: false, stdout: '', stderr: local.error };
+        return withIdentity(request.target, (target) =>
+          streamBackup(target, request.producerCommand, local.path, runner, request.opts ?? {}),
+        );
+      }
+
+      case 'streamRestore': {
+        const local = containedPath(request.localPath, readableRoots, 'read');
+        if ('error' in local) return { ok: false, stdout: '', stderr: local.error };
+        return withIdentity(request.target, (target) =>
+          streamRestore(target, local.path, request.consumerCommand, runner, request.opts ?? {}),
+        );
+      }
+
+      case 'installAuthorizedKey':
+        return withIdentity(request.target, (target) =>
+          installAuthorizedKey(target, request.password, runner, request.opts ?? {}),
+        );
+    }
+  }
+
+  await new Promise<void>((resolvePromise, reject) => {
+    server.once('error', reject);
+    server.listen(socketPath, resolvePromise);
+  });
+
+  return {
+    socketPath,
+    stop: () => {
+      stopped = true;
+    },
+    close: () => {
+      stopped = true;
+      server.close();
+    },
+  };
+}
+
+/** A refusal in the shape the refused op's caller reads. */
+function refusal(op: RemoteRequest['op'], message: string): object {
+  return op === 'checkTarget' ? { allowed: false, message } : failure(message);
+}
+
+function failure(message: string): { ok: false; stdout: ''; stderr: string } {
+  return { ok: false, stdout: '', stderr: message };
+}
+
+/**
+ * Materialise the module's own key for the one call (task 5.4), then remove
+ * it. Content in, path out: the asking half read the bytes inside the jail,
+ * and the file below exists only for ssh's `-i`, which takes no stdin.
+ */
+function withIdentity(target: BridgeTarget, run: (target: RemoteTarget) => RunResult): RunResult {
+  const base: RemoteTarget = {
+    ipv4_address: target.ipv4_address,
+    ...(target.user !== undefined ? { user: target.user } : {}),
+    ...(target.port !== undefined ? { port: target.port } : {}),
+  };
+  if (target.identityContent === undefined) return run(base);
+
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-hook-identity-'));
+  const keyPath = join(dir, 'key');
+  try {
+    writeFileSync(keyPath, target.identityContent, { mode: 0o600 });
+    return run({ ...base, identityFile: keyPath });
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/**
+ * Confine a stream primitive's LOCAL path to this run's granted roots.
+ *
+ * Every component is resolved through the filesystem before the containment
+ * test. For a read, `realpath` of the whole path does that. For a write the
+ * leaf may not exist yet, so the PARENT is realpath'd — and a leaf that does
+ * exist as a symlink is refused outright: the shell's `>` follows it, so a
+ * link planted inside a writable directory (`state/evil` →
+ * `/var/celilo/master.key`, dangling inside the jail, resolving outside it)
+ * would otherwise aim celilo's write anywhere. Refusing the link beats
+ * resolving it, because the hook's own subprocesses could re-point it
+ * between a check and the write.
+ *
+ * The mount set binds the same path inside and outside the jail, which is
+ * what makes a contained path name the same file for both sides — and why a
+ * path OUTSIDE the roots (a bare `/tmp/x` under the run's private tmpfs)
+ * must be refused: celilo would write a file the hook then cannot even see.
+ */
+function containedPath(
+  localPath: string,
+  roots: readonly string[],
+  access: 'read' | 'write',
+): { path: string } | { error: string } {
+  let real: string;
+  try {
+    if (access === 'read') {
+      real = realpathSync(resolve(localPath));
+    } else {
+      const resolved = resolve(localPath);
+      real = join(realpathSync(dirname(resolved)), basename(resolved));
+      // lstat, not existsSync: a dangling link (its target absent inside the
+      // jail) still IS a link, and that is the case that aims celilo's write
+      // out of the jail. existsSync follows the link and misses it.
+      if (isSymlink(real)) {
+        return {
+          error: `remote-ops broker: local path '${localPath}' is a symlink; a stream write follows links, so it must name the file itself.`,
+        };
+      }
+    }
+  } catch (error) {
+    return {
+      error: `remote-ops broker: local path '${localPath}' is not usable (${error instanceof Error ? error.message : String(error)})`,
+    };
+  }
+
+  if (roots.some((root) => real === root || real.startsWith(root + sep))) return { path: real };
+  return {
+    error: `remote-ops broker: local path '${localPath}' is outside this run's ${access === 'write' ? 'writable' : 'readable'} directories. Use ctx.stateDir or a declared path input (backup_dir, restore_dir).`,
+  };
+}
+
+function realpathOrSelf(path: string): string {
+  try {
+    return realpathSync(resolve(path));
+  } catch {
+    return resolve(path);
+  }
+}
+
+/** True if `path` itself is a symlink, dangling target included. */
+function isSymlink(path: string): boolean {
+  try {
+    return lstatSync(path).isSymbolicLink();
+  } catch {
+    return false;
+  }
+}

package/src/hooks/run-named-hook.ts

@@ -23,6 +23,7 @@ import {
   listDnsRegistrations,
   stampDnsRegistrationsRefreshed,
 } from '../services/dns-registrations';
+import { remoteAccessPolicy } from '../services/remote-access';
 import { loadCapabilityFunctions } from './capability-loader';
 import { invokeHook } from './executor';
 import { loadHookConfigMap } from './load-hook-config';
@@ -190,6 +191,7 @@ export async function runNamedHook(
       capabilities: capabilityFunctions,
       requiredCapabilities,
       systems: getModuleSystems(moduleId, db),
+      remoteAccess: remoteAccessPolicy(moduleId, db),
       timeoutMs: options.timeoutMs,
     },
   );

package/src/hooks/test-fixtures/jail-probe-hook.ts

@@ -8,7 +8,9 @@
  * whether the access worked and hands the message back for the post-mortem.
  */
 
-import { readFileSync, writeFileSync } from 'node:fs';
+import { readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
 import { defineHook } from '@celilo/capabilities';
 
 function probe(fn: () => void): { succeeded: boolean; detail: string } {
@@ -54,6 +56,17 @@ export default defineHook({
       staged_write: probe(() => {
         writeFileSync(`${config.staged_input}/produced`, 'staged input survived the tmpfs');
       }),
+      // Stage 3's observable (design D12, task 5.5): the SSH credential is
+      // not bound, so a hook cannot authenticate anywhere by hand. Reads
+      // `homedir()/.ssh` — the exact path the pre-stage-3 jail bound (via
+      // `join(homedir(), '.ssh')`), so the red baseline reaches a planted key
+      // and this branch does not. `homedir()` rather than `process.env.HOME`
+      // because that is what the mount decision used, and the two can differ.
+      ssh_key: probe(() => {
+        const sshDir = join(homedir(), '.ssh');
+        const names = readdirSync(sshDir);
+        for (const name of names) readFileSync(join(sshDir, name));
+      }),
     };
   },
 });

package/src/hooks/test-fixtures/jail-toolchain-hook.ts

@@ -0,0 +1,227 @@
+/**
+ * Test fixture: what TOOLING can a hook reach from inside the jail (task 4.13)?
+ *
+ * `jail-probe-hook.ts` next door asks what a hook can reach on the FILESYSTEM,
+ * and D9's table is written in those terms. This one asks the question D9's
+ * table does not answer: the fleet's real hooks shell out. `remote.ts` builds an
+ * `ssh <user>@<host> <cmd>` string and hands it to `execSync`, which spawns
+ * `/bin/sh -c`. Neither `/bin/sh` nor `/usr/bin/ssh` appears anywhere in the
+ * derivation, and 22 module script files import `node:child_process`.
+ *
+ * So every probe here reports an outcome rather than throwing, the same way its
+ * neighbour does. The point is to MEASURE the reach rather than reason about it:
+ * reading the derivation is what produced the belief that stage 2 was fine, and
+ * reading it again would produce the same belief.
+ *
+ * `bwrap` is probed too, and its expected answer is the opposite of every other
+ * one here. It must NOT be reachable. D9 predicts exactly how that gets undone —
+ * a hook fails on a missing binary, somebody binds the directory it lives in,
+ * and the escape path reopens with a green suite — so the guard belongs in the
+ * same run as the failures that would tempt someone into it.
+ */
+
+import { execFileSync, execSync } from 'node:child_process';
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { BROWSER_ROOT, defineHook } from '@celilo/capabilities';
+
+interface Probe {
+  succeeded: boolean;
+  detail: string;
+}
+
+function probe(fn: () => string): Probe {
+  try {
+    return { succeeded: true, detail: fn().slice(0, 200) };
+  } catch (error) {
+    return { succeeded: false, detail: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+async function probeAsync(fn: () => Promise<string>): Promise<Probe> {
+  try {
+    return { succeeded: true, detail: (await fn()).slice(0, 200) };
+  } catch (error) {
+    return { succeeded: false, detail: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+/**
+ * Launch args for the two arms of task 4.10's controlled experiment, exported
+ * so the unit test that pins the flag on the launch path asserts the SAME
+ * definition the probes launch with rather than a copy of it.
+ *
+ * `SANDBOXED_LAUNCH_ARGS` is the experiment arm: Chromium's own sandbox on,
+ * which does not start inside the jail (its namespace sandbox cannot nest in
+ * the unprivileged user namespace bwrap creates). It must stay sandboxed or
+ * the comparison stops being one that isolates a variable.
+ *
+ * `JAILED_LAUNCH_ARGS` is the launch path itself, and the interim celilo#1215
+ * approved: a Chromium launched by a jailed hook starts with `--no-sandbox`,
+ * because the jail is then the only sandbox around the browser. The flag
+ * belongs here, in the launcher — never in the mount set or any other
+ * contract-shaped place. Brokered rendering outside the jail (v2, celilo#1215)
+ * is the preferred end state and removes the need for this flag.
+ */
+export const SANDBOXED_LAUNCH_ARGS: readonly string[] = ['--headless', '--dump-dom', 'about:blank'];
+export const JAILED_LAUNCH_ARGS: readonly string[] = [
+  '--headless',
+  '--no-sandbox',
+  '--dump-dom',
+  'about:blank',
+];
+
+export default defineHook({
+  hook: 'container_created',
+  requires: [],
+  handler: async (ctx) => {
+    // Through `ctx.config`, not an env var. Stage 1 stopped forwarding the
+    // process environment into hooks (celilo#1158), so a probe keyed on
+    // `process.env` would read empty and report "no browser supplied" for a
+    // run that supplied one — a false ABSENT indistinguishable from a real one.
+    const browser = (ctx.config as { browser_executable?: string }).browser_executable ?? '';
+
+    return {
+      // The one every other shell-out depends on. `execSync` spawns
+      // `/bin/sh -c`, so a jail without it cannot run ANY hook that uses
+      // `node:child_process`, whatever binary that hook was reaching for.
+      shell: probe(() => execSync('echo alive', { encoding: 'utf-8' })),
+
+      // Bypasses the shell to separate two failures that look identical
+      // through `execSync`: no `/bin/sh`, versus no target binary.
+      exec_without_shell: probe(() => execFileSync('/bin/echo', ['alive'], { encoding: 'utf-8' })),
+
+      // What `remote.ts` needs. Stage 2 binds `~/.ssh` read-only ON PURPOSE so
+      // that remote.ts keeps working, which makes the absence of the binary it
+      // feeds that key to the interesting result rather than a detail.
+      ssh: probe(() => execFileSync('/usr/bin/ssh', ['-V'], { encoding: 'utf-8', stdio: 'pipe' })),
+
+      // The deploy path. `ansible-playbook` is a Python program, so a bind of
+      // the binary alone would not be enough even if one existed.
+      ansible: probe(() =>
+        execFileSync('/usr/bin/ansible-playbook', ['--version'], { encoding: 'utf-8' }),
+      ),
+
+      // Name resolution, in two halves, because they fail for different
+      // reasons and only one of them is about the mount set.
+      resolv_conf: probe(() => readFileSync('/etc/resolv.conf', 'utf-8')),
+      dns_lookup: await probeAsync(async () => {
+        // Bun's own resolver, so this measures name resolution rather than the
+        // absence of some binary that happens to do it. A jail with no
+        // `/etc/resolv.conf` has no nameserver to ask.
+        // Bounded, because the interesting failure is a HANG. With no
+        // `/etc/resolv.conf` there is no nameserver to ask and the resolver
+        // waits out its own retry schedule, so an unbounded probe reports the
+        // test runner's timeout instead of the jail's behaviour.
+        const answer = await Promise.race([
+          Bun.dns.lookup('example.com', { family: 4 }),
+          new Promise<never>((_, reject) =>
+            setTimeout(() => reject(new Error('no answer within 3s')), 3_000),
+          ),
+        ]);
+        return answer[0]?.address ?? 'resolved, no address';
+      }),
+
+      // Files a resolver reads before it ever touches the network.
+      nsswitch: probe(() => readFileSync('/etc/nsswitch.conf', 'utf-8')),
+      hosts_file: probe(() => readFileSync('/etc/hosts', 'utf-8')),
+
+      // ── Task 4.10, the browser case ──────────────────────────────────
+      //
+      // The task says to bind `~/.cache/ms-playwright`. That path is stale:
+      // `managed-browser-runtime` has since landed `BROWSER_ROOT` in
+      // `@celilo/capabilities`, so a provisioned browser lives under it and
+      // `resolveBrowser()` is what a hook asks for one. Both are probed, and
+      // only the playwright cache is ASSERTED — see the suite for why.
+      playwright_cache: probe(() => {
+        const home = process.env.HOME ?? '/root';
+        if (existsSync(`${home}/.cache/ms-playwright`)) return 'present';
+        throw new Error(`no ${home}/.cache/ms-playwright`);
+      }),
+      celilo_browser_root: probe(() => {
+        if (existsSync(BROWSER_ROOT)) return `present at ${BROWSER_ROOT}`;
+        throw new Error(`no ${BROWSER_ROOT} on this host`);
+      }),
+
+      // ⚠️ BROWSER_ROOT is a SUBDIRECTORY of celilo's Linux data directory
+      // (`/var/lib/celilo`), which also holds `celilo.db` and `master.key`.
+      // Binding it must not expose its siblings. bubblewrap creates the parent
+      // as an empty directory in the jail's namespace, so `..` reaches
+      // nothing — but that is the kind of claim worth a probe rather than a
+      // sentence, because it is the whole acceptance criterion D9 satisfies
+      // by absence.
+      data_dir_sibling: probe(() => {
+        const parent = dirname(BROWSER_ROOT);
+        return readFileSync(`${parent}/master.key`, 'utf-8');
+      }),
+
+      // Why a present binary can still fail to exec. `posix_spawn` reports a
+      // missing ELF interpreter as ENOENT on the PROGRAM, which is
+      // indistinguishable from the program being absent. Probed rather than
+      // inferred, and worth it: the loader turned out to be present and the
+      // first hypothesis was wrong.
+      dynamic_loader: probe(() => {
+        const found: string[] = [];
+        for (const dir of ['/lib', '/lib64', '/usr/lib']) {
+          try {
+            for (const name of readdirSync(dir)) {
+              if (name.startsWith('ld-linux') || name.startsWith('ld.so')) {
+                found.push(`${dir}/${name}`);
+              }
+            }
+          } catch {
+            // A directory the derivation legitimately skipped: `/lib64` does
+            // not exist on arm64 and `planJailedSpawn` drops a missing source.
+          }
+        }
+        if (found.length === 0) {
+          throw new Error('no ld-linux loader under /lib, /lib64 or /usr/lib');
+        }
+        return found.join(', ');
+      }),
+
+      // Did the bind happen at all? Separates "no browser reached the jail"
+      // from "one did and will not run", which the two launches below cannot
+      // tell apart on their own.
+      browser_binary_present: probe(() => {
+        if (!browser) throw new Error('no browser supplied to this run');
+        if (existsSync(browser)) return `present at ${browser}`;
+        throw new Error(`absent at ${browser}`);
+      }),
+
+      // The nesting question, and the only pair here needing a real browser.
+      //
+      // Two launches differing in ONE flag, so the result is a controlled
+      // experiment rather than an observation. Chromium's own sandbox forks a
+      // helper into a NEW user namespace, and nesting that inside
+      // bubblewrap's unprivileged one is what D9 and task 4.10 both predict
+      // will fail. `--no-sandbox` skips the fork, so the launch that must
+      // succeed carries it (celilo#1215 interim). The args come from the
+      // exported constants above, which the flag-pinning unit test shares.
+      browser_sandboxed: probe(() => {
+        if (!browser) throw new Error('no browser supplied to this run');
+        return execFileSync(browser, [...SANDBOXED_LAUNCH_ARGS], {
+          encoding: 'utf-8',
+          stdio: 'pipe',
+          timeout: 20_000,
+        });
+      }),
+      browser_no_sandbox: probe(() => {
+        if (!browser) throw new Error('no browser supplied to this run');
+        return execFileSync(browser, [...JAILED_LAUNCH_ARGS], {
+          encoding: 'utf-8',
+          stdio: 'pipe',
+          timeout: 20_000,
+        });
+      }),
+
+      // MUST be false. See the docblock.
+      bwrap_present: probe(() => {
+        for (const path of ['/usr/bin/bwrap', '/usr/local/bin/bwrap', '/bin/bwrap']) {
+          if (existsSync(path)) return `present at ${path}`;
+        }
+        throw new Error('no bwrap on any known path');
+      }),
+    };
+  },
+});