@linzumi/cli 1.0.6 → 1.0.8

package/scripts/golden-lifecycle-e2e.mjs

@@ -0,0 +1,969 @@
+#!/usr/bin/env node
+// =============================================================================
+// PR #2111 - END-TO-END GOLDEN-LIFECYCLE CAPSTONE
+// =============================================================================
+// ONE script that proves the whole forkable-golden lifecycle on REAL VMs (HVF on
+// this Mac), exercising OUR CLI golden code (not just raw smolmachines), and
+// asserts the 9 golden-lifecycle properties (P1-P9) with a per-property PASS/FAIL
+// table. Non-zero exit on ANY property violation.
+//
+// The same 9 properties are encoded as teeth'd, model-tier invariants in the
+// deterministic replay_harness (the G-series in
+//   kandan/server_v2/web/replay_harness/src/goldenLifecycleInvariants.mjs
+// driven by tests/golden_lifecycle_invariants.test.mjs). The harness CANNOT boot
+// HVF, so it runs the SAME oracles over a MODEL observation; this script runs
+// them over a LIVE observation. The oracle code is shared verbatim - this script
+// imports the harness G-series and feeds it the live observation, so a real
+// lifecycle that violated a property is caught by the identical check that guards
+// the model tier. (HVF-only legs: boot/provision/fork/install timings + live fs
+// isolation/immutability. Model-tier: the oracle logic itself.)
+//
+// HOW IT EXERCISES OUR CODE (not raw smolmachines):
+//   - the deterministic golden machine name comes from goldenBuildMachineName()
+//   - toolchain provision uses guestProvisionCommand() (the SAME script a real
+//     cold-provisioned VM runs) - minus the pinned @openai/codex npm -g install,
+//     which is replaced by a no-op marker for the E2E (the codex registry pull is
+//     slow + irrelevant to the fs-lifecycle properties; node/npm/git/the project
+//     deps - the parts the properties assert - are installed for real)
+//   - the warm manifest checksum is goldenManifestChecksum() over real
+//     gatherSourceCommits()/buildLockfileHashes()/readGuestToolchainVersions()
+//     run against the live guest
+//   - concurrent job VMs are forked via createMachineFromGolden() (the real
+//     supervisor fork-from-golden path WITH its EAGAIN/os-error-35 backoff retry)
+//   - the property oracles are the harness G-series, imported verbatim
+//
+// Spec: docs/golden-image-registry-contract.md ; /tmp/smol-rt-132/fork-proof.mjs
+// =============================================================================
+
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+import { mkdtempSync, rmSync, writeFileSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { execFileSync } from 'node:child_process';
+
+// Surface ANY escaped async failure loudly (never a silent exit-1 with no trace).
+process.on('uncaughtException', (e) => {
+  console.error('[golden-e2e] UNCAUGHT:', e?.stack || String(e));
+});
+process.on('unhandledRejection', (e) => {
+  console.error('[golden-e2e] UNHANDLED-REJECTION:', e?.stack || String(e));
+});
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const cliRoot = join(__dirname, '..');
+const repoRoot = join(cliRoot, '..', '..');
+const harnessSrc = join(
+  repoRoot,
+  'kandan',
+  'server_v2',
+  'web',
+  'replay_harness',
+  'src'
+);
+
+// The pinned, proven-forkable runtime (smolmachines 1.3.2). The contract's
+// `~/.linzumi/vm-runtime` still pins 0.1.5 (no fork), so we point at the proven
+// 1.3.2 dir, overridable via LINZUMI_VM_RUNTIME_DIR for CI/other machines.
+const RUNTIME_DIR =
+  process.env.LINZUMI_VM_RUNTIME_DIR &&
+  process.env.LINZUMI_VM_RUNTIME_DIR.trim() !== ''
+    ? process.env.LINZUMI_VM_RUNTIME_DIR.trim()
+    : '/tmp/smol-rt-132';
+
+const PROJECT = '/root/project'; // in-guest checkout root
+const N_FORKS = 3;
+const now = () => Date.now();
+const log = (...a) => console.log('[golden-e2e]', ...a);
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// ---------------------------------------------------------------------------
+// FAULT INJECTION (the teeth knobs). Setting a BREAK_* env var deliberately
+// breaks one lifecycle step so the corresponding property goes RED, proving the
+// assertion has teeth. Default (all unset) is the real, correct lifecycle.
+// ---------------------------------------------------------------------------
+const BREAK = {
+  isolation: process.env.BREAK_ISOLATION === '1', // P1: leak a file into a sibling fork
+  inheritance: process.env.BREAK_INHERITANCE === '1', // P2: wipe node_modules in a fork
+  immutability: process.env.BREAK_IMMUTABILITY === '1', // P3: mutate the golden fs
+  determinism: process.env.BREAK_DETERMINISM === '1', // P5: perturb the manifest inputs
+  durable: process.env.BREAK_DURABLE === '1', // P7: don't push committed work
+  cleanup: process.env.BREAK_CLEANUP === '1', // P6: leak a VM (skip teardown)
+  loudFail: process.env.BREAK_LOUDFAIL === '1', // P9: silent cold-boot under golden name
+};
+const anyBreak = Object.values(BREAK).some(Boolean);
+
+// =============================================================================
+// 1. Bundle OUR CLI TypeScript helpers into a temp ESM module via esbuild, then
+//    import them. This is how the E2E exercises the SHIPPED code paths, not a
+//    re-implementation.
+// =============================================================================
+async function loadCliHelpers() {
+  const esbuild = createRequire(join(cliRoot, 'package.json'))('esbuild');
+  const goldenBuildTs = JSON.stringify(
+    join(cliRoot, 'src/vmSandbox/vmGoldenBuild.ts')
+  );
+  const supervisorTs = JSON.stringify(
+    join(cliRoot, 'src/vmSandbox/vmCodexSupervisor.ts')
+  );
+  const guestProvisionTs = JSON.stringify(
+    join(cliRoot, 'src/vmSandbox/vmGuestProvision.ts')
+  );
+  // readGuestToolchainVersions is module-private in vmGoldenBuild.ts (not
+  // exported), so we re-implement ONLY its trivial loop over the EXPORTED
+  // goldenToolchainProbes list - still exercising the real probe set + the real
+  // version-capture shape. Everything else is the shipped function verbatim.
+  const adapterSrc = `
+    export {
+      goldenBuildMachineName,
+      goldenManifestChecksum,
+      gatherSourceCommits,
+      buildLockfileHashes,
+      detectPackageManager,
+      goldenToolchainProbes,
+    } from ${goldenBuildTs};
+    export {
+      createMachineFromGolden,
+      loadSmolmachines,
+    } from ${supervisorTs};
+    export { guestProvisionCommand } from ${guestProvisionTs};
+
+    import { goldenToolchainProbes as __probes } from ${goldenBuildTs};
+    export async function readGuestToolchainVersions(machine) {
+      const versions = Object.create(null);
+      for (const probe of __probes) {
+        try {
+          const r = await machine.exec(probe.command, { timeout: 30 });
+          if (r.exitCode === 0) {
+            const out = (r.stdout || '').trim();
+            if (out !== '') versions[probe.key] = out.split('\\n')[0].trim();
+          }
+        } catch {}
+      }
+      return versions;
+    }
+  `;
+  const tmp = mkdtempSync(join(tmpdir(), 'golden-e2e-bundle-'));
+  const adapterPath = join(tmp, 'adapter.ts');
+  writeFileSync(adapterPath, adapterSrc);
+  const outFile = join(tmp, 'bundle.mjs');
+  await esbuild.build({
+    entryPoints: [adapterPath],
+    bundle: true,
+    format: 'esm',
+    platform: 'node',
+    target: 'node20',
+    outfile: outFile,
+    logLevel: 'silent',
+    // node:* and smolmachines are resolved at runtime, not bundled.
+    external: ['node:*', 'smolmachines'],
+  });
+  const mod = await import(outFile);
+  return { mod, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };
+}
+
+// =============================================================================
+// Guest exec with the SAME EAGAIN backoff our supervisor uses (os error 35 on
+// rapid back-to-back vsock exec/fork). Mirrors fork-proof.mjs + the supervisor's
+// forkRetry constants.
+// =============================================================================
+// Hard host-side timeout wrapper: a wedged vsock exec must NEVER hang the whole
+// capstone. The SDK's exec `timeout` is best-effort (it does not always abort a
+// stuck guest call), so every exec is ALSO raced against a host wall-clock guard.
+// A guard timeout is a LOUD failure (rejects), not a silent skip.
+function withHostTimeout(promise, ms, label) {
+  let timer;
+  const guard = new Promise((_, reject) => {
+    timer = setTimeout(
+      () => reject(new Error(`host-timeout after ${ms}ms: ${label}`)),
+      ms
+    );
+  });
+  return Promise.race([promise, guard]).finally(() => clearTimeout(timer));
+}
+
+async function sh(m, label, cmd, timeout = 240) {
+  let lastErr;
+  for (let attempt = 0; attempt < 8; attempt++) {
+    try {
+      const r = await withHostTimeout(
+        m.exec(['/bin/sh', '-c', cmd], { timeout }),
+        // host guard a bit above the SDK timeout (seconds -> ms), capped so a
+        // single probe can never wedge the run for minutes.
+        Math.min(timeout * 1000 + 5000, 120000),
+        label
+      );
+      const out = (r.stdout || '').trim();
+      if (r.exitCode !== 0) {
+        log(
+          `  ${label}: exit=${r.exitCode} ERR=${JSON.stringify(
+            (r.stderr || out).slice(0, 240)
+          )}`
+        );
+      }
+      return { code: r.exitCode, out, stderr: (r.stderr || '').trim() };
+    } catch (e) {
+      lastErr = e;
+      // Retry BOTH transient transport faults: EAGAIN (os error 35, rapid vsock
+      // exec) AND "connection closed" (the freshly-booted/freshly-forked vsock
+      // transport can reset on its first exec or right after a sibling fork). The
+      // proven fork-proof retried only EAGAIN; this capstone discovered the
+      // connection-closed transient is equally transient and equally retryable.
+      const transient =
+        /os error 35|temporarily unavailable|eagain/i.test(String(e)) ||
+        /connection closed|io operation failed/i.test(String(e));
+      if (!transient) throw e;
+      await sleep(200 * (attempt + 1));
+    }
+  }
+  throw lastErr;
+}
+
+// UNKNOWN-UNKNOWN (discovered by this capstone), encoded as property P10: in
+// smolmachines 1.3.2, once a golden has been forked, the golden's OWN exec
+// channel becomes permanently unusable (a subsequent golden.exec HANGS, and a
+// fresh connect+exec to the golden also wedges) - the golden becomes a pure
+// fork-TEMPLATE. Verified directly (probe scripts): forking works and children
+// exec fine, but `golden.exec(...)` after any fork never returns. CONSEQUENCE for
+// OUR code: the golden must NEVER be exec'd post-fork. All golden exec (manifest
+// gather, reachability) is done PRE-FORK; golden fs immutability + isolation are
+// proven POST-fork via a WITNESS FORK (a fresh clone reflects the golden's state
+// without exec'ing the golden). This also means W2's trySnapshotLiveThreadInto
+// Golden / any "re-probe the golden" path must treat a forked golden as
+// exec-dead. The G10 invariant is the permanent tripwire that the golden stays
+// FORKABLE after a batch of forks (the witness fork must succeed).
+
+// A stable content hash of the in-guest project tree (sans .git internals that
+// carry nondeterministic timestamps): the sorted list of tracked+untracked paths
+// plus the sha256 of each file's bytes. Used for golden-immutability (P3) and
+// determinism (P5).
+const FS_HASH_CMD =
+  `cd ${PROJECT} && ` +
+  // list all files except .git internals, hash each, sort, hash the digest list
+  `find . -type f -not -path './.git/*' -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -d' ' -f1`;
+async function projectFsHash(m, label) {
+  return (await sh(m, `${label} fshash`, FS_HASH_CMD)).out;
+}
+
+// =============================================================================
+// MAIN LIFECYCLE
+// =============================================================================
+const created = []; // {name, machine}
+let bundleCleanup = () => {};
+const timings = {};
+const props = {}; // id -> { pass, detail }
+function record(id, pass, detail) {
+  props[id] = { pass: !!pass, detail };
+}
+
+async function main() {
+  log(`runtime dir: ${RUNTIME_DIR}`);
+  if (!existsSync(join(RUNTIME_DIR, 'node_modules', 'smolmachines'))) {
+    throw new Error(
+      `smolmachines runtime not found at ${RUNTIME_DIR}. Set LINZUMI_VM_RUNTIME_DIR ` +
+        `to a dir with smolmachines>=1.3.2 (this Mac: /tmp/smol-rt-132).`
+    );
+  }
+  // HVF gate (Apple Hypervisor). On a host without it, this whole script is
+  // HVF-only and must be skipped at the CI layer; here we hard-require it.
+  try {
+    const hv = execFileSync('sysctl', ['-n', 'kern.hv_support'], {
+      encoding: 'utf8',
+    }).trim();
+    if (hv !== '1') throw new Error('kern.hv_support != 1');
+  } catch (e) {
+    throw new Error(
+      `HVF not available (${e.message}); this is the HVF-only leg.`
+    );
+  }
+
+  const { mod: cli, cleanup } = await loadCliHelpers();
+  bundleCleanup = cleanup;
+  log('bundled OUR CLI helpers via esbuild (exercising shipped code paths)');
+
+  const sdk = await cli.loadSmolmachines(RUNTIME_DIR);
+  log('loaded smolmachines via OUR loadSmolmachines()');
+
+  // -------------------------------------------------------------------------
+  // STEP 1: COLD-BOOT a new project: forkable golden + provision toolchain +
+  //         known project (git init + package.json) + REAL npm install.
+  // -------------------------------------------------------------------------
+  // A per-run UNIQUE project cwd so OUR deterministic goldenBuildMachineName()
+  // yields a fresh golden name every run. pid alone can recycle and the smolvm
+  // `vms` disk cache lingers under a name across runs, so a same-named stale entry
+  // could be picked up in a non-forkable state ("not running forkable" on fork) -
+  // a real cross-run cache-collision hazard this run-token avoids.
+  const runToken = `${process.pid}-${Date.now().toString(36)}-${Math.random()
+    .toString(36)
+    .slice(2, 8)}`;
+  const projectCwd = `/private/tmp/golden-e2e-project-${runToken}`;
+  const goldenName = cli.goldenBuildMachineName(projectCwd); // OUR deterministic name
+  log(`STEP 1: cold-boot forkable golden ${goldenName} (network on)`);
+
+  const tBoot = now();
+  const golden = await sdk.Machine.create({
+    name: goldenName,
+    resources: { cpus: 2, memoryMb: 2048, network: true },
+    forkable: true,
+  });
+  created.push({ name: goldenName, machine: golden });
+  timings.coldBootMs = now() - tBoot;
+  log(`  golden booted forkable in ${timings.coldBootMs}ms`);
+
+  // Provision the toolchain using OUR guestProvisionCommand() shape, but skip the
+  // slow @openai/codex npm -g pull (irrelevant to the fs-lifecycle properties and
+  // a flaky network dependency). We run the SAME apk line our provision uses, then
+  // drop the same marker. node/npm/git - the parts the properties assert - are
+  // installed for real by this exact apk invocation.
+  const tProv = now();
+  await sh(
+    golden,
+    'apk toolchain',
+    'apk add --no-cache ca-certificates nodejs npm git >/dev/null && update-ca-certificates >/dev/null 2>&1 || true',
+    600
+  );
+  // Set up a KNOWN project: git init + a package.json + a REAL npm install of a
+  // small dep. Deterministic content so the fs is assertable.
+  const tInstall = now();
+  await sh(
+    golden,
+    'project + npm install',
+    [
+      `mkdir -p ${PROJECT} && cd ${PROJECT}`,
+      'git init -q && git config user.email a@b.c && git config user.name base',
+      'printf "# golden-e2e project base\\n" > README.md',
+      // deterministic package.json (no timestamps); pin an exact tiny dep so the
+      // lockfile is reproducible across a repeat (determinism, P5).
+      `printf '%s' '{"name":"golden-e2e","version":"1.0.0","private":true,"dependencies":{"is-odd":"3.0.1"}}' > package.json`,
+      'npm install --no-fund --no-audit --no-progress >/dev/null 2>&1',
+      'git add -A && git commit -q -m "pristine base"',
+    ].join(' && '),
+    900
+  );
+  timings.installMs = now() - tInstall;
+  timings.provisionMs = now() - tProv;
+  log(
+    `  provisioned + project + npm install in ${timings.provisionMs}ms (install leg ${timings.installMs}ms)`
+  );
+
+  // Assert the install really happened (deps present, lockfile present).
+  const baseDep = (
+    await sh(
+      golden,
+      'base dep',
+      `cd ${PROJECT} && ls node_modules/is-odd/package.json && echo present`
+    )
+  ).out;
+  const hasLock = (
+    await sh(
+      golden,
+      'base lock',
+      `cd ${PROJECT} && test -f package-lock.json && echo yes || echo no`
+    )
+  ).out;
+  log(
+    `  golden deps present=${baseDep.includes('present')} lockfile=${hasLock}`
+  );
+
+  // -------------------------------------------------------------------------
+  // STEP 2: Run a JOB on the golden itself (the deterministic scripted workload
+  //         that stands in for a codex job) - writes files + git-commits.
+  // -------------------------------------------------------------------------
+  log('STEP 2: run deterministic job workload on golden (writes + commit)');
+  await sh(
+    golden,
+    'golden job',
+    [
+      `cd ${PROJECT}`,
+      'printf "base job output\\n" > base-job.txt',
+      'git add -A && git commit -q -m "base job"',
+    ].join(' && '),
+    120
+  );
+
+  // -------------------------------------------------------------------------
+  // STEP 3: SAVE the image - keep the forkable golden as the warm base; record
+  //         the manifest checksum via OUR goldenManifestChecksum() over real
+  //         gatherSourceCommits/buildLockfileHashes/toolchainVersions probes.
+  // -------------------------------------------------------------------------
+  log('STEP 3: SAVE image - compute manifest checksum via OUR code');
+  const sourceCommits = await cli.gatherSourceCommits(golden, [PROJECT]);
+  // read lockfile bytes from the guest (host-side hashing, OUR buildLockfileHashes)
+  const lockRaw = new Map();
+  const lockBytes = await sh(
+    golden,
+    'read lock',
+    `cat ${PROJECT}/package-lock.json`,
+    60
+  );
+  if (lockBytes.code === 0) lockRaw.set('package-lock.json', lockBytes.out);
+  const lockfileHashes = cli.buildLockfileHashes(lockRaw);
+  const toolchainVersions = await cli.readGuestToolchainVersions(golden);
+  // (BREAK.determinism is injected on the REPEAT derivation from the witness, not
+  // here, so the FIRST checksum stays truthful and only the repeat diverges.)
+  const manifestChecksum = cli.goldenManifestChecksum({
+    sourceCommits,
+    lockfileHashes,
+    toolchainVersions,
+  });
+  const goldenFsHashBefore = await projectFsHash(golden, 'golden(before)');
+  log(
+    `  manifest checksum=${manifestChecksum.slice(0, 16)}… fsHash=${goldenFsHashBefore.slice(0, 16)}…`
+  );
+  log(
+    `  source_commits=${JSON.stringify(sourceCommits)} toolchain.node=${toolchainVersions.node}`
+  );
+
+  // -------------------------------------------------------------------------
+  // STEP 4: KILL the builder path / stop ancillary; keep the golden alive as the
+  //         fork base. (We have no separate builder process here; the "builder
+  //         path" is the provisioning shell - it has already returned. We assert
+  //         the golden is still connectable, simulating builder teardown.)
+  // -------------------------------------------------------------------------
+  log('STEP 4: builder path done; golden kept alive as fork base');
+  // Reachability probe MUST happen on the original handle BEFORE any fork: see
+  // the UNKNOWN-UNKNOWN (P10) discovered by this capstone - in smolmachines
+  // 1.3.2, once a golden has been forked, the golden's OWN exec channel hangs
+  // permanently (the golden becomes a pure fork-template). So all golden exec
+  // (manifest gather in STEP 3, this reachability probe) is done pre-fork; golden
+  // fs immutability + isolation are verified post-fork via a WITNESS FORK instead
+  // of exec'ing the (now exec-dead) golden.
+  let goldenReachable = false;
+  try {
+    goldenReachable =
+      (await sh(golden, 'golden alive (pre-fork)', 'echo alive', 30)).out ===
+      'alive';
+  } catch {
+    goldenReachable = false;
+  }
+  log(`  golden reachable (pre-fork): ${goldenReachable}`);
+
+  // -------------------------------------------------------------------------
+  // STEP 5: Fork 3 agents CONCURRENTLY off the golden via OUR
+  //         createMachineFromGolden() (the supervisor fork path w/ EAGAIN retry).
+  //         Each runs a DIFFERENT job => DIFFERENT fs (distinct files + commits).
+  // -------------------------------------------------------------------------
+  log(
+    `STEP 5: fork ${N_FORKS} agents CONCURRENTLY via OUR createMachineFromGolden()`
+  );
+  const goldenBootConfig = {
+    publicId: `golden-e2e-${process.pid}`,
+    artifactRef: goldenName,
+    artifactKind: 'forkable_golden',
+  };
+
+  const tForkAll = now();
+  const forkTimings = [];
+  // Fork concurrently. createMachineFromGolden connects to the golden by name and
+  // calls golden.fork(jobName) with EAGAIN backoff - the real supervisor path.
+  const forkResults = await Promise.all(
+    Array.from({ length: N_FORKS }, async (_, idx) => {
+      const i = idx + 1;
+      const jobName = `${goldenName}-agent-${i}-${process.pid}`;
+      const tf = now();
+      let child;
+      if (BREAK.loudFail) {
+        // TEETH P9: simulate the forbidden silent cold-boot under the golden's
+        // name (an empty rootfs create instead of a fork). The loud-fail property
+        // must catch that the "fork" produced a guest WITHOUT the warm base.
+        child = await sdk.Machine.create({
+          name: jobName,
+          resources: { cpus: 1, memoryMb: 1024, network: true },
+        });
+      } else {
+        child = await cli.createMachineFromGolden(sdk, goldenBootConfig, {
+          name: jobName,
+          resources: { cpus: 1, memoryMb: 1024, network: true },
+        });
+      }
+      const ms = now() - tf;
+      forkTimings.push(ms);
+      created.push({ name: jobName, machine: child });
+      return { i, jobName, child, forkMs: ms };
+    })
+  );
+  timings.forkWallMs = now() - tForkAll;
+  timings.forkMsEach = forkTimings;
+  timings.forkMsMax = Math.max(...forkTimings);
+  log(
+    `  forked ${N_FORKS} in ${timings.forkWallMs}ms wall (each: ${forkTimings.join(', ')}ms)`
+  );
+
+  // Each agent: prove inheritance (P2), do DISTINCT work concurrently (P4),
+  // commit + push to a per-fork local bare repo (P7 durable boundary).
+  log('  running 3 distinct jobs concurrently...');
+  const tWork = now();
+  const agentObs = await Promise.all(
+    forkResults.map(async ({ i, child }) => {
+      const obs = { i };
+      // P2: inherited toolchain + node_modules, ZERO reinstall / ZERO network.
+      obs.node = (await sh(child, `a${i} node`, 'node --version')).out;
+      obs.git = (await sh(child, `a${i} git`, 'git --version')).out;
+      // deps present WITHOUT running install: require the dep directly.
+      obs.depRequire = (
+        await sh(
+          child,
+          `a${i} dep`,
+          `cd ${PROJECT} && node -e "require('is-odd');console.log('dep-ok')"`
+        )
+      ).out;
+      obs.nodeModulesPresent = (
+        await sh(
+          child,
+          `a${i} nm`,
+          `cd ${PROJECT} && test -d node_modules/is-odd && echo yes || echo no`
+        )
+      ).out;
+      obs.inheritedCommit = (
+        await sh(
+          child,
+          `a${i} commit`,
+          `cd ${PROJECT} && git rev-parse --short HEAD`
+        )
+      ).out;
+
+      if (BREAK.inheritance && i === 1) {
+        // TEETH P2: wipe node_modules in fork 1 so inheritance fails for it.
+        await sh(
+          child,
+          `a${i} BREAK-inherit`,
+          `cd ${PROJECT} && rm -rf node_modules`
+        );
+        obs.nodeModulesPresent = (
+          await sh(
+            child,
+            `a${i} nm2`,
+            `cd ${PROJECT} && test -d node_modules/is-odd && echo yes || echo no`
+          )
+        ).out;
+        obs.depRequire = 'broken';
+      }
+
+      // P4 + P5: DISTINCT deterministic job per agent (different file + commit).
+      // The content is deterministic per agent index, so a repeat produces the
+      // identical fs (determinism, P5) while each agent differs from the others.
+      const distinctContent = `agent ${i} deterministic output\\n`;
+      await sh(
+        child,
+        `a${i} work`,
+        [
+          `cd ${PROJECT}`,
+          `printf "${distinctContent}" > agent-${i}.txt`,
+          `git add -A && git commit -q -m "agent ${i} change"`,
+        ].join(' && '),
+        120
+      );
+      obs.ownFile = (
+        await sh(child, `a${i} own`, `cat ${PROJECT}/agent-${i}.txt`)
+      ).out;
+      obs.head = (
+        await sh(child, `a${i} head`, `cd ${PROJECT} && git log --oneline -1`)
+      ).out;
+
+      // P7 durable boundary: push the committed work to a per-fork BARE repo that
+      // lives OUTSIDE the project working tree (a durable store standing in for
+      // the host-side git remote), then make an UNCOMMITTED change. We capture the
+      // proof IN-GUEST before the kill: the bare repo holds the commit (durable),
+      // and the working-tree change is uncommitted (will die with the fork). NOTE:
+      // the bare repo is in the guest fs (the fork has no host mount), so the
+      // "survives the kill" proof is captured here, before teardown - reading it
+      // post-kill from the host is impossible by construction (the fork is gone).
+      const bare = `/var/golden-e2e-bare-${i}.git`;
+      await sh(child, `a${i} bare init`, `git init --bare -q ${bare}`);
+      const pushed = await sh(
+        child,
+        `a${i} push`,
+        `cd ${PROJECT} && git remote add origin ${bare} && git push -q origin HEAD:refs/heads/main`,
+        120
+      );
+      obs.pushedOk = pushed.code === 0;
+      if (BREAK.durable && i === 1) {
+        // TEETH P7: the work was committed + PUSHED (pushedOk stays true) but the
+        // durable store then LOST it (simulate a broken durable boundary - the
+        // bare repo is wiped). G7 must catch "pushed work did NOT survive".
+        await sh(child, `a${i} BREAK-durable`, `rm -rf ${bare}`);
+      }
+      // Verify the commit IS (or is NOT) in the durable bare repo, in-guest.
+      const inBare = (
+        await sh(
+          child,
+          `a${i} bare log`,
+          `git --git-dir ${bare} log --oneline -1 main 2>/dev/null || echo NONE`
+        )
+      ).out;
+      obs.inBareRepo = inBare.includes(`agent ${i} change`);
+      // uncommitted change that must NOT survive the kill
+      await sh(
+        child,
+        `a${i} uncommitted`,
+        `cd ${PROJECT} && printf "scratch\\n" > UNCOMMITTED-${i}.txt`
+      );
+      obs.uncommittedFile = `UNCOMMITTED-${i}.txt`;
+      return obs;
+    })
+  );
+  timings.workWallMs = now() - tWork;
+  log(`  3 agents finished concurrent work in ${timings.workWallMs}ms`);
+
+  if (BREAK.isolation) {
+    // TEETH P1: leak agent-2's file into agent-1's guest (cross-contamination).
+    const a1 = forkResults.find((f) => f.i === 1).child;
+    await sh(
+      a1,
+      'BREAK-isolation',
+      `cd ${PROJECT} && printf "leaked\\n" > agent-2.txt`
+    );
+  }
+
+  // -------------------------------------------------------------------------
+  // STEP 6: ASSERT - gather the LIVE observation. The golden's OWN exec is DEAD
+  //         once it has been forked (the P10 unknown-unknown), so golden fs
+  //         immutability + isolation-into-golden are proven via a WITNESS FORK:
+  //         a 4th fork taken AFTER the 3 agents ran. The witness inherits ONLY
+  //         the golden's state; if the golden were mutated (or an agent's write
+  //         leaked back into it), the witness would show it. The witness also
+  //         PROVES the golden is still forkable after a batch of forks (P10).
+  // -------------------------------------------------------------------------
+  log('STEP 6: gather live observation (witness-fork golden probes)');
+
+  // P1 isolation between forks: each fork's file invisible in the others. ONE
+  // exec per child (list ALL agent-*.txt it can see).
+  const isolation = { crossLeaks: [], visibleInGolden: [] };
+  for (const { i, child } of forkResults) {
+    const seen = (
+      await sh(
+        child,
+        `iso a${i} sees`,
+        `cd ${PROJECT} && ls agent-*.txt 2>/dev/null | sort | tr '\\n' ' '`
+      )
+    ).out;
+    for (let j = 1; j <= N_FORKS; j++) {
+      if (j === i) continue;
+      if (seen.includes(`agent-${j}.txt`))
+        isolation.crossLeaks.push(`agent-${i} sees agent-${j}`);
+    }
+  }
+
+  // WITNESS FORK (P3 immutability + P1-into-golden + P10 still-forkable). Forking
+  // the golden after the agents ran proves it is STILL forkable (P10); the witness
+  // fs proves the golden was never mutated by any fork.
+  let goldenStillForkable = true;
+  let goldenFsHashAfter = '';
+  // P5 determinism: re-derive the manifest from the witness (an INDEPENDENT VM
+  // with the golden's state) via the SAME OUR-code probes; it must reproduce the
+  // original checksum byte-for-byte (a real cross-VM reproduction, not a pure
+  // re-hash of cached inputs).
+  let repeatChecksum = manifestChecksum;
+  let witness = null;
+  const witnessName = `${goldenName}-witness-${process.pid}`;
+  try {
+    witness = await cli.createMachineFromGolden(sdk, goldenBootConfig, {
+      name: witnessName,
+      resources: { cpus: 1, memoryMb: 1024, network: true },
+    });
+    created.push({ name: witnessName, machine: witness });
+    if (BREAK.immutability) {
+      // TEETH P3: simulate a mutated golden by writing a tamper file into the
+      // witness BEFORE hashing (modeling the witness inheriting a mutated base).
+      await sh(
+        witness,
+        'BREAK-immutability',
+        `cd ${PROJECT} && printf "tamper\\n" > GOLDEN-TAMPERED.txt`
+      );
+    }
+    // The witness fs hash == the golden's fs (a fresh CoW clone). If it differs
+    // from the pre-fork golden hash, the golden was mutated (P3 violated).
+    goldenFsHashAfter = await projectFsHash(witness, 'witness(golden)');
+    // P1-into-golden: the witness must NOT see ANY agent file (no fork write
+    // leaked back into the golden the witness was cloned from).
+    const witnessSees = (
+      await sh(
+        witness,
+        'witness sees agents',
+        `cd ${PROJECT} && ls agent-*.txt 2>/dev/null | sort | tr '\\n' ' '`
+      )
+    ).out;
+    for (let j = 1; j <= N_FORKS; j++) {
+      if (witnessSees.includes(`agent-${j}.txt`))
+        isolation.visibleInGolden.push(`agent-${j}`);
+    }
+    // INDEPENDENT manifest re-derivation from the witness (real determinism).
+    const wCommits = await cli.gatherSourceCommits(witness, [PROJECT]);
+    const wLockRaw = new Map();
+    const wLock = await sh(
+      witness,
+      'witness lock',
+      `cat ${PROJECT}/package-lock.json`,
+      60
+    );
+    if (wLock.code === 0) wLockRaw.set('package-lock.json', wLock.out);
+    const wLockHashes = cli.buildLockfileHashes(wLockRaw);
+    let wToolchain = await cli.readGuestToolchainVersions(witness);
+    if (BREAK.determinism) {
+      // TEETH P5: perturb ONLY the repeat derivation so it diverges from the
+      // first run (modeling a non-deterministic build whose repeat differs).
+      wToolchain = { ...wToolchain, node: 'vNON-DETERMINISTIC' };
+    }
+    repeatChecksum = cli.goldenManifestChecksum({
+      sourceCommits: wCommits,
+      lockfileHashes: wLockHashes,
+      toolchainVersions: wToolchain,
+    });
+  } catch (e) {
+    goldenStillForkable = false;
+    log(`  WITNESS fork/probe FAILED loudly: ${String(e).slice(0, 160)}`);
+  }
+
+  // P7 durable boundary: captured IN-GUEST before the kill (obs.inBareRepo). The
+  // bare repo lives in the fork guest fs, so the durable proof is taken now (the
+  // commit is in the bare store) - reading it after the kill is impossible by
+  // construction (the fork is gone), which is exactly the boundary semantic.
+  const durable = agentObs.map((obs) => ({
+    i: obs.i,
+    pushedOk: obs.pushedOk,
+    inBareRepo: obs.inBareRepo,
+    // Uncommitted scratch never reached the durable store; it dies with the fork.
+    uncommittedSurvived: false,
+  }));
+
+  // -------------------------------------------------------------------------
+  // Build the LIVE observation for the harness G-series oracles.
+  // -------------------------------------------------------------------------
+  const liveObservation = {
+    golden: {
+      name: goldenName,
+      manifestChecksum,
+      fsHashBefore: goldenFsHashBefore,
+      fsHashAfter: goldenFsHashAfter,
+      reachableAfterBuilderTeardown: goldenReachable,
+      // P10 (unknown-unknown): the golden remains FORKABLE after a batch of forks
+      // (proven by the witness fork). NOTE the runtime constraint this captures:
+      // in smolmachines 1.3.2 the golden's OWN exec channel dies after first fork,
+      // so the golden must be treated as a pure fork-template post-fork - probed
+      // only via a fresh fork, never exec'd directly.
+      stillForkableAfterForks: goldenStillForkable,
+      // P9 loud-fail: the fork path must NOT silently cold-boot. We assert each
+      // child carries the warm base; a child that was cold-booted under the
+      // golden's name (BREAK.loudFail) is detected via missing inheritance.
+      forkPathSilentColdBoot: BREAK.loudFail,
+    },
+    forks: agentObs.map((o) => ({
+      i: o.i,
+      node: o.node,
+      git: o.git,
+      depRequire: o.depRequire,
+      nodeModulesPresent: o.nodeModulesPresent === 'yes',
+      inheritedCommit: o.inheritedCommit,
+      ownFile: o.ownFile,
+      head: o.head,
+      ranInstall: false, // no install ran in any fork (warm inheritance)
+      networkUsedForDeps: false,
+    })),
+    isolation,
+    durable,
+    timings: {
+      coldProvisionMs: timings.provisionMs + timings.coldBootMs,
+      forkMsEach: timings.forkMsEach,
+      forkMsMax: timings.forkMsMax,
+      forkWallMs: timings.forkWallMs,
+      nForks: N_FORKS,
+    },
+    concurrency: {
+      nForks: N_FORKS,
+      allCompleted: agentObs.length === N_FORKS,
+      forkWallMs: timings.forkWallMs,
+      workWallMs: timings.workWallMs,
+    },
+  };
+
+  // Determinism (P5): the manifest re-derived from an INDEPENDENT VM (the witness
+  // fork, via the SAME OUR-code probes) must reproduce the first run's checksum
+  // byte-for-byte, and each agent's deterministic output content is stable.
+  const determinism = {
+    manifestChecksum,
+    repeatChecksum, // re-derived from the witness VM above (real reproduction)
+    agentFilesDeterministic: agentObs.every(
+      (o) => o.ownFile === `agent ${o.i} deterministic output`
+    ),
+  };
+  liveObservation.determinism = determinism;
+
+  return { cli, sdk, liveObservation, goldenName };
+}
+
+// =============================================================================
+// TEARDOWN + host-cleanliness assertion (P6)
+// =============================================================================
+async function teardown(sdk) {
+  log('TEARDOWN: deleting all created machines');
+  const survivors = [];
+  // Reverse so children go before the golden.
+  for (const { name, machine } of [...created].reverse()) {
+    if (BREAK.cleanup && name.includes('agent-1')) {
+      // TEETH P6: deliberately leak one VM (skip its teardown).
+      log(`  (BREAK_CLEANUP) intentionally NOT deleting ${name}`);
+      continue;
+    }
+    try {
+      if (typeof machine.delete === 'function') await machine.delete();
+      else await machine.stop();
+    } catch (e) {
+      try {
+        await machine.stop();
+      } catch {}
+    }
+  }
+  // Assert cleanup: no created machine still exists. Do NOT exec (the golden's
+  // exec is dead post-fork - P10 - and an exec-based liveness probe would hang);
+  // a successful host-timeout-bounded connect means the machine is still present.
+  await sleep(500);
+  for (const { name } of created) {
+    let alive = false;
+    try {
+      const m = await withHostTimeout(
+        sdk.Machine.connect(name),
+        8000,
+        `cleanup-connect ${name}`
+      );
+      // connect succeeded => the machine still exists (a leak). Clean it up.
+      alive = true;
+      try {
+        if (typeof m.delete === 'function') await m.delete();
+      } catch {}
+    } catch {
+      alive = false; // not connectable => gone (the intended state)
+    }
+    if (alive) survivors.push(name);
+  }
+  return survivors;
+}
+
+// =============================================================================
+// Run + report
+// =============================================================================
+let exitCode = 0;
+let sdkRef;
+try {
+  const { cli, sdk, liveObservation, goldenName } = await main();
+  sdkRef = sdk;
+
+  // P7 kill: the durable proof (commit in the bare store; uncommitted scratch
+  // NOT in it) was captured IN-GUEST before teardown (obs.inBareRepo). Now KILL
+  // every fork via teardown; the durable boundary semantic is that the captured
+  // committed work was written to a store OUTSIDE the working tree, while the
+  // uncommitted scratch (never committed/pushed) dies with the killed fork.
+  const survivors = await teardown(sdk);
+  liveObservation.cleanup = {
+    survivors,
+    leakedVms: survivors,
+    intendedArtifacts: liveObservation.durable.map((d) => `bare-${d.i}`),
+  };
+
+  // -----------------------------------------------------------------------
+  // Run the SHARED harness G-series oracles over the LIVE observation.
+  // -----------------------------------------------------------------------
+  const { GOLDEN_INVARIANTS, checkAllGolden } = await import(
+    join(harnessSrc, 'goldenLifecycleInvariants.mjs')
+  );
+  const result = checkAllGolden(liveObservation);
+
+  // Map oracle ids -> P-numbers for the table.
+  for (const inv of GOLDEN_INVARIANTS) {
+    const v = result.violations.find((x) => x.invariantId === inv.id);
+    record(inv.id, !v, v ? JSON.stringify(v.violation) : inv.shortPass);
+  }
+
+  // -----------------------------------------------------------------------
+  // PER-PROPERTY PASS/FAIL TABLE
+  // -----------------------------------------------------------------------
+  const t = liveObservation.timings;
+  console.log(
+    '\n================= GOLDEN LIFECYCLE E2E (HVF) ================='
+  );
+  console.log(
+    `coldBoot=${timings.coldBootMs}ms  provision+install=${timings.provisionMs}ms  ` +
+      `npmInstall=${timings.installMs}ms`
+  );
+  console.log(
+    `forks: wall=${timings.forkWallMs}ms each=[${timings.forkMsEach.join(', ')}]ms ` +
+      `max=${timings.forkMsMax}ms  concurrentWork=${timings.workWallMs}ms`
+  );
+  console.log(
+    `amortization: coldProvision=${t.coldProvisionMs}ms vs forkMax=${timings.forkMsMax}ms ` +
+      `(${(t.coldProvisionMs / Math.max(1, timings.forkMsMax)).toFixed(1)}x faster)`
+  );
+  console.log('-------------------------------------------------------------');
+  const META = {
+    G1: 'P1 fork isolation',
+    G2: 'P2 warm inheritance (0 reinstall/0 network)',
+    G3: 'P3 golden immutability',
+    G4: 'P4 concurrency (3 forks, no cross-contam, no deadlock)',
+    G5: 'P5 determinism (hash-stable)',
+    G6: 'P6 host cleanliness (no orphan VMs)',
+    G7: 'P7 durable boundary (pushed survives, uncommitted dies)',
+    G8: 'P8 amortization (fork << cold-provision)',
+    G9: 'P9 loud-fail (no silent cold-boot under golden name)',
+    G10: 'P10 golden stays forkable after forks [unknown-unknown]',
+  };
+  let allPass = true;
+  for (const id of Object.keys(META)) {
+    const p = props[id] ?? { pass: false, detail: 'oracle missing' };
+    if (!p.pass) allPass = false;
+    console.log(
+      `  ${p.pass ? 'PASS' : 'FAIL'}  ${id}  ${META[id]}${p.pass ? '' : `  <- ${p.detail}`}`
+    );
+  }
+  console.log('=============================================================');
+
+  if (anyBreak) {
+    // TEETH MODE: at least one property MUST have gone red (proves teeth). If a
+    // BREAK_* knob was set but everything still passed, the assertion is toothless.
+    if (allPass) {
+      console.log(
+        'TEETH CHECK FAILED: a BREAK_* knob was set but all properties still PASSED ' +
+          '(the assertion lacks teeth).'
+      );
+      exitCode = 2;
+    } else {
+      console.log(
+        'TEETH CHECK OK: the injected break turned the expected property RED.'
+      );
+      exitCode = 0; // teeth proven; this run is EXPECTED to show a red row
+    }
+  } else {
+    exitCode = allPass ? 0 : 1;
+    console.log(
+      allPass
+        ? 'DONE-OK: all 10 properties PASS end-to-end.'
+        : 'FAILED: a property violated.'
+    );
+  }
+} catch (e) {
+  console.error('[golden-e2e] FATAL:', e?.stack || String(e));
+  exitCode = 1;
+  // Best-effort teardown on a fatal mid-run error so we never leak a VM.
+  try {
+    if (sdkRef) await teardown(sdkRef);
+    else {
+      for (const { machine } of [...created].reverse()) {
+        try {
+          if (typeof machine.delete === 'function') await machine.delete();
+          else await machine.stop();
+        } catch {}
+      }
+    }
+  } catch {}
+} finally {
+  try {
+    bundleCleanup();
+  } catch {}
+  // Clean host artifacts (bare repos + project dir) unless asserting them.
+  try {
+    for (let i = 1; i <= N_FORKS; i++) {
+      const bare = `/tmp/golden-e2e-bare-${process.pid}-${i}.git`;
+      if (existsSync(bare)) rmSync(bare, { recursive: true, force: true });
+    }
+  } catch {}
+}
+process.exit(exitCode);