@bongos/core 1.17.3 → 1.18.3

package/scripts/gds/package-core.js

@@ -52,8 +52,15 @@
 //
 // USAGE:
 //   node scripts/gds/package-core.js [--ref <git-ref>] [--out-dir dist]
-//        [--version <v>] [--droplet-ip <ip>] [--publish [--live]]
+//        [--version <v>] [--droplet-ip <ip>] [--self-verify [--self-verify-boot]] [--publish [--live]]
 //   Defaults: --ref HEAD, --out-dir dist, --version = platform CORE_VERSION.
+//   --self-verify    after building, prove the artifact reproduces its pin (unpack the .tgz,
+//                    recompute tree_sha256 == manifest + verify tarball.sha256). Fatal on mismatch.
+//                    (task 1002218 / audit H3) — deterministic + network-free.
+//   --self-verify-boot   ALSO `npm install <tgz>` into a throwaway consumer + boot /healthz
+//                    (reuses tests/consumer_layout_boot.mjs; needs npm/registry access).
+//   --leak-scan-strict   make the post-build leak scan FATAL on any prior-instance literal
+//                    (default: the scan runs + WARNS; --no-leak-scan skips it). task 1002220 / H6.
 //   Outputs (nothing is committed — dist/ is gitignored):
 //     dist/bongos-core-<version>/           the redacted core tree (inspectable)
 //     dist/bongos-core-<version>.tgz        the installable unit
@@ -64,6 +71,7 @@
 //                    npm org on a paid plan + `npm login`; this script holds no token.
 
 const fs = require('node:fs');
+const os = require('node:os');
 const path = require('node:path');
 const zlib = require('node:zlib');
 const crypto = require('node:crypto');
@@ -207,6 +215,33 @@ function processCoreFile({ pathRel, raw, rules, sensitiveLiterals, envSecrets })
   };
 }
 
+// ---------------------------------------------------------------------------
+// npm-STRIPPED special files (idea 1000543) — the file-SELECTION gate that keeps
+// the pin HONEST about what a consumer actually installs.
+//
+// `npm install <tgz>` never materializes certain special files into the installed
+// package — most notably `.gitignore` (confirmed live: installing the 1.18.1 core
+// .tgz drops exactly one file, ./.gitignore), plus `.npmignore` and `.npmrc`. But
+// package-core enumerates the GIT tree, so isPublishable() happily selects a tracked
+// `.gitignore`. If it lands in the manifest files[] + tree_sha256 pin, the tree a
+// consumer recomputes over node_modules/@bongos/core is MISSING it → verifyInstalledPin
+// (upgrade.js) reports `missing installed file: .gitignore` → hash mismatch → the bump
+// auto-rolls-back. That breaks EVERY vendored `bongos upgrade` with pin-verify ON, and
+// the unattended auto-upgrade subscription (task 2150 / ADR 0136) has no way to pass
+// --skip-pin-verify — so it would roll back every patch (the fix reported in idea 1000543).
+//
+// The manifest must describe exactly what npm installs, so these files are dropped BEFORE
+// they enter the tarball, filesMeta, or the pin (main(), below). Matched by BASENAME at any
+// depth — npm strips nested ones too. The asymmetry is deliberately safe: excluding a file
+// npm WOULD have kept only slightly weakens the pin (verifyInstalledPin iterates the manifest
+// files[], so an un-pinned file is simply not checked); FAILING to exclude one npm strips is
+// FATAL. And these files are useless inside a vendored dependency anyway.
+const NPM_STRIPPED_BASENAMES = new Set(['.gitignore', '.npmignore', '.npmrc']);
+function isNpmStripped(pathRel) {
+  const base = String(pathRel || '').split('/').pop();
+  return NPM_STRIPPED_BASENAMES.has(base);
+}
+
 // ---------------------------------------------------------------------------
 // The CORE's own package.json + package-lock.json (task 1882, ADR 0108 §2).
 //
@@ -545,6 +580,164 @@ function publishTarball({ tgzPath, live }) {
   console.error(`[package-core] npm publish ${live ? 'succeeded — @bongos/core is live (private).' : 'dry-run OK — the tarball is publishable.'}`);
 }
 
+// The neutral FLOOR — customer-zero identity tokens a portable core must never carry (task 1002220
+// / H6 + task 1002217 / H4, finding F2). When packaging from the NEUTRAL trunk there is no
+// config/branding.json, so the branding-DERIVED redaction + gate see ZERO literals and report
+// "clean" while stripping nothing. This fixed floor closes that gap two ways, both keyed off ONE
+// definition:
+//   • DOC path (H4, fail-closed): injected as `extra` {tokens, owners} into buildRedactionRules +
+//     collectSensitiveLiterals, so the existing doc-redaction STRIPS these from published docs
+//     (tokens→NEUTRAL_TOKEN, owners→NEUTRAL_OWNER) and the fail-closed no-leak gate ABORTS the build
+//     if one SURVIVES — even with no branding.json. The floor lives HERE and is injected at call
+//     time, so mirror-redact.js stays literal-free (its self-leak invariant holds); functional files
+//     still ship VERBATIM (the R88 fix) and are covered by the H6 advisory below, not this gate.
+//   • SCAN path (H6, advisory): scanArtifactForLeaks measures the flat list across the WHOLE artifact.
+// findLeaks' public-IP floor separately catches the old droplet IP.
+const PRIOR_INSTANCE_FLOOR = { tokens: ['amazonprimea', 'off the boats'], owners: ['lars589'] };
+const PRIOR_INSTANCE_LITERALS = [...PRIOR_INSTANCE_FLOOR.tokens, ...PRIOR_INSTANCE_FLOOR.owners];
+
+// High-signal committed-credential shapes for the secret WARNING (task 1002220 / audit H6, finding
+// F3). Deliberately NARROW — well-known provider token prefixes + PEM private-key headers + an
+// explicit secret=<literal> assignment — rather than the generic entropy catch-all, which fires on
+// ordinary long identifiers / base64 data and buries a real hit (an early cut warned on 339 files,
+// ~all benign). A match is a WARNING to eyeball; the HARD gate against live env-secret VALUES is
+// functionalSecretLeaks in the build loop. Scans functional files (docs are redacted + gated already).
+const CREDENTIAL_SHAPE_PATTERNS = [
+  ['anthropic-key', /\bsk-ant-[A-Za-z0-9_-]{16,}/],
+  ['openai-key', /\bsk-[A-Za-z0-9]{24,}/],
+  ['github-token', /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}\b|\bgithub_pat_[A-Za-z0-9_]{20,}/],
+  ['aws-access-key-id', /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/],
+  ['google-api-key', /\bAIza[A-Za-z0-9_-]{30,}/],
+  ['slack-token', /\bxox[baprs]-[A-Za-z0-9-]{10,}/],
+  ['private-key-pem', /-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----/],
+  ['secret-assignment', /\b(?:api[_-]?key|secret|password|passwd|access[_-]?token|auth[_-]?token)\b\s*[:=]\s*['"][^'"\s]{16,}['"]/i],
+];
+function credentialShapeHits(text) {
+  const names = [];
+  for (const [name, re] of CREDENTIAL_SHAPE_PATTERNS) if (re.test(text)) names.push(name);
+  return names;
+}
+
+// classify a shipped path for the scan — mirrors processCoreFile's doc/stub/functional split.
+function scanKindOf(pathRel) {
+  if (isAgentDoc(pathRel)) return 'stub';
+  if (isDocPath(pathRel)) return 'doc';
+  return 'functional';
+}
+
+// Independent post-build leak scan over the built core file set (task 1002220 / audit H6). Two layers:
+//   • INSTANCE LITERALS — findLeaks() over PRIOR_INSTANCE_LITERALS (+ the public-IP floor) across
+//     EVERY shipped file: brand/owner/history that must not ship in a portable core. This is the
+//     "prove no leak" measurement the neutral trunk was missing (F2).
+//   • CREDENTIAL SHAPES — credentialShapeHits() over FUNCTIONAL files only (docs are redacted+gated
+//     already): a high-signal WARNING that a committed credential shape shipped.
+// `files` = [{ path, content }]. Pure — returns findings; the caller decides warn vs fatal.
+function scanArtifactForLeaks(files, { literals = PRIOR_INSTANCE_LITERALS } = {}) {
+  const literalHits = []; // { path, label, sample }
+  const secretWarnings = []; // { path, shapes }
+  for (const f of files || []) {
+    const text = Buffer.isBuffer(f.content) ? f.content.toString('utf8') : String(f.content == null ? '' : f.content);
+    for (const lk of findLeaks(text, literals)) literalHits.push({ path: f.path, label: lk.label, sample: lk.sample });
+    if (scanKindOf(f.path) === 'functional') {
+      const shapes = credentialShapeHits(text);
+      if (shapes.length) secretWarnings.push({ path: f.path, shapes });
+    }
+  }
+  return { literalHits, secretWarnings };
+}
+
+// Self-verify the built artifact (task 1002218 / audit H3, findings F1+F5). The packager
+// claims a consumer "verifies an install by recomputing [the pin] from the unpacked tree" —
+// this proves the packager's OWN output actually does, at build time, on the produce side:
+//   1. REPRODUCE THE PIN (always; deterministic, network-free): unpack the built .tgz and
+//      recompute tree_sha256 over the unpacked tree, asserting it equals the manifest — plus
+//      the on-disk .tgz bytes vs manifest.tarball.sha256. Catches a tar-writer bug, a corrupt
+//      write, or pin drift BEFORE the artifact is trusted or published (the R88 failure class).
+//   2. BARE-INSTALL BOOT (opt-in --self-verify-boot; needs npm/registry): `npm install <tgz>`
+//      into a throwaway consumer and boot /healthz, by reusing the proven consumer-layout
+//      harness (tests/consumer_layout_boot.mjs) with CORE_TGZ set — no duplication. The deeper
+//      standing CI gate is task 1002221; this is the packager's own quick smoke.
+// Fatal (exit 1) on a reproduce mismatch or a boot failure — a self-verify that finds a
+// problem must stop the build, never warn-and-ship.
+// Pure recompute — read each pinned file from an unpacked tree, rebuild the canonical tree
+// hash (using the manifest's already-canonical modes), and compare to the manifest, plus the
+// on-disk .tgz bytes vs manifest.tarball.sha256. No process.exit / no spawn → unit-testable.
+// Returns { ok, problems, fileCount }.
+function reproducePinFromTree({ treeRoot, sidecar, tgzPath }, fsImpl = fs) {
+  const problems = [];
+  const recomputed = [];
+  for (const f of sidecar.files || []) {
+    const abs = path.join(treeRoot, ...String(f.path).split('/'));
+    let buf;
+    try { buf = fsImpl.readFileSync(abs); } catch { problems.push(`missing in artifact: ${f.path}`); continue; }
+    const sha = sha256hex(buf);
+    if (sha !== f.sha256) problems.push(`content mismatch: ${f.path}`);
+    recomputed.push({ path: f.path, mode: f.mode, sha256: sha });
+  }
+  const treeHash = canonicalTreeHash(recomputed);
+  if (problems.length === 0 && treeHash !== sidecar.tree_sha256) {
+    problems.push(`recomputed tree_sha256 ${treeHash.slice(0, 12)}… ≠ manifest ${String(sidecar.tree_sha256).slice(0, 12)}…`);
+  }
+  if (tgzPath && sidecar.tarball && sidecar.tarball.sha256) {
+    const tgzSha = sha256hex(fsImpl.readFileSync(tgzPath));
+    if (tgzSha !== sidecar.tarball.sha256) {
+      problems.push(`on-disk .tgz sha256 ${tgzSha.slice(0, 12)}… ≠ manifest.tarball.sha256 ${String(sidecar.tarball.sha256).slice(0, 12)}…`);
+    }
+  }
+  return { ok: problems.length === 0, problems, fileCount: (sidecar.files || []).length };
+}
+
+function selfVerify({ tgzPath, sidecar, outDir, boot }) {
+  console.error('');
+  console.error('[self-verify] verifying the built artifact reproduces its pin …');
+
+  // 1. reproduce the pin from the built .tgz.
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'bongos-selfverify-'));
+  try {
+    let treeRoot;
+    const x = spawnSync('tar', ['-xzf', tgzPath, '-C', tmp], { encoding: 'utf8' });
+    if (!x.error && x.status === 0) {
+      treeRoot = path.join(tmp, TAR_ROOT.replace(/\/$/, '')); // the `package/` root inside the tarball
+    } else {
+      // tar unavailable — fall back to the on-disk stage dir (the SAME staged buffers written
+      // to the .tgz), so the pin is still recomputed from a materialized tree. Less faithful
+      // (won't catch a tar-writer bug), so it's a loud warning, not silent.
+      treeRoot = path.join(outDir, stageDirName(sidecar.core_version));
+      console.error(`[self-verify] ! tar unavailable (${x.error ? x.error.message : 'exit ' + x.status}); recomputing over the stage dir ${path.relative(REPO_ROOT, treeRoot)} instead of the unpacked .tgz.`);
+      if (!fs.existsSync(treeRoot)) { console.error('[self-verify] FAILED — neither `tar` nor the stage dir is available to recompute the pin.'); process.exit(1); }
+    }
+    const rep = reproducePinFromTree({ treeRoot, sidecar, tgzPath });
+    if (!rep.ok) {
+      console.error(`[self-verify] FAILED — the built artifact does NOT reproduce its pin (${rep.problems.length} problem(s)):`);
+      for (const p of rep.problems.slice(0, 20)) console.error('   ' + p);
+      if (rep.problems.length > 20) console.error(`   … +${rep.problems.length - 20} more`);
+      process.exit(1);
+    }
+    console.error(`[self-verify] ✓ pin reproduced — tree_sha256 + tarball.sha256 match the manifest (${rep.fileCount} files).`);
+  } finally {
+    try { fs.rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ }
+  }
+
+  // 2. bare-install boot (opt-in) — reuse the proven consumer-layout harness with CORE_TGZ.
+  if (boot) {
+    const harness = path.join(REPO_ROOT, 'tests', 'consumer_layout_boot.mjs');
+    if (!fs.existsSync(harness)) {
+      console.error('[self-verify] • bare-install boot skipped — tests/consumer_layout_boot.mjs not present (packaging from an installed core, where tests/ is not shipped).');
+      return;
+    }
+    console.error('[self-verify] bare-install boot: `npm install <tgz>` + boot /healthz (via consumer_layout_boot, CORE_TGZ set) …');
+    const r = spawnSync(process.execPath, [harness], {
+      cwd: REPO_ROOT, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024,
+      env: { ...process.env, CORE_TGZ: tgzPath }, stdio: ['ignore', process.stderr, process.stderr],
+    });
+    if (r.error || r.status !== 0) {
+      console.error(`[self-verify] FAILED — the packaged core did not install + boot (${r.error ? r.error.message : 'exit ' + r.status}).`);
+      process.exit(1);
+    }
+    console.error('[self-verify] ✓ bare-install boot OK — the packaged core installs and serves /healthz.');
+  }
+}
+
 function main() {
   const ref = arg('--ref', 'HEAD');
   const outDir = path.resolve(REPO_ROOT, arg('--out-dir', 'dist'));
@@ -564,8 +757,17 @@ function main() {
   //    dependency set, not the game's. Pulled out of the generic per-file loop below.
   const SYNTHESIZED_PATHS = new Set(['package.json', 'package-lock.json']);
   const entries = parseLsTree(git(['ls-tree', '-r', sha])).filter((e) => e.type === 'blob');
-  const publishable = entries.filter((e) => isPublishable(e.path) && !SYNTHESIZED_PATHS.has(e.path));
+  const selected = entries.filter((e) => isPublishable(e.path) && !SYNTHESIZED_PATHS.has(e.path));
+  // Drop npm-never-installed files (.gitignore, .npmignore, .npmrc) so the manifest +
+  // tree_sha256 pin describe exactly what `npm install` materializes — otherwise the
+  // consumer's verifyInstalledPin fails on the missing file and the vendored upgrade
+  // auto-rolls-back (idea 1000543). Done here, so they never reach the tarball or the pin.
+  const npmStripped = selected.filter((e) => isNpmStripped(e.path));
+  const publishable = selected.filter((e) => !isNpmStripped(e.path));
   console.error(`[package-core] ${ref} = ${sha.slice(0, 12)} — ${entries.length} files, ${publishable.length} in the core (+2 synthesized), ${entries.length - publishable.length - 2} excluded.`);
+  if (npmStripped.length) {
+    console.error(`[package-core] dropped ${npmStripped.length} npm-never-installed file(s) from the pin (npm strips these on install; idea 1000543): ${npmStripped.map((e) => e.path).join(', ')}`);
+  }
   if (publishable.length === 0) {
     console.error('refusing: the core set is EMPTY — almost certainly a manifest/ref mismatch, not intentional. Aborting.');
     process.exit(1);
@@ -595,11 +797,19 @@ function main() {
   try { branding = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'config/branding.json'), 'utf8')); }
   catch { /* vanilla instance: derive from the public-IP floor only */ }
   const ownerAliases = String(process.env.OWNER_ALIASES || '').split(',').map((s) => s.trim()).filter(Boolean);
-  const extra = { owners: ownerAliases, literals: dropletIp ? [dropletIp] : [] };
+  // The NEUTRAL FLOOR (task 1002217 / audit H4): always inject the fixed prior-instance tokens so
+  // docs de-brand + the no-leak gate fires against them EVEN WITH NO branding.json (the F2 gap).
+  // Merged on top of this instance's own branding — for a branded instance the floor is harmless
+  // extra; for the neutral trunk it is the only thing standing between a doc and customer-zero's name.
+  const extra = {
+    owners: [...ownerAliases, ...PRIOR_INSTANCE_FLOOR.owners],
+    tokens: [...PRIOR_INSTANCE_FLOOR.tokens],
+    literals: dropletIp ? [dropletIp] : [],
+  };
   const rules = buildRedactionRules(branding, extra);
   const sensitiveLiterals = collectSensitiveLiterals(branding, extra);
   const envSecrets = collectEnvSecrets();
-  console.error(`[package-core] ${rules.length} redaction rule(s), ${sensitiveLiterals.length} gate literal(s) + public-IP floor + ${envSecrets.length} env-secret literal(s).`);
+  console.error(`[package-core] ${rules.length} redaction rule(s), ${sensitiveLiterals.length} gate literal(s) + public-IP floor + ${envSecrets.length} env-secret literal(s) (incl. the neutral floor).`);
 
   // 4. Process every file IN MEMORY (processCoreFile: docs redacted, functional verbatim)
   //    + the fail-closed no-leak gate. Nothing touches dist/ until the ENTIRE set is proven
@@ -710,6 +920,47 @@ function main() {
   console.error(`  manifest       ${path.relative(REPO_ROOT, manifestPath)}   ← the PIN a downstream repo records`);
   console.error(`  tree           ${path.relative(REPO_ROOT, stageDir)}/`);
 
+  // Optional self-verify leg (task 1002218 / audit H3): prove the built artifact reproduces
+  // its pin (+ optionally installs & boots) BEFORE it can be trusted or published. Runs after
+  // the artifact is written; a failure exits non-zero so an unverified/broken artifact never
+  // reaches the publish leg below or a downstream consumer.
+  if (process.argv.includes('--self-verify')) {
+    selfVerify({ tgzPath, sidecar, outDir, boot: process.argv.includes('--self-verify-boot') });
+  }
+
+  // Post-build leak scan (task 1002220 / audit H6): independent verification that the BUILT
+  // artifact carries no prior-instance identity, + a warn-level secret-shape scan over functional
+  // files. Runs on every build (skip with --no-leak-scan). Prior-instance literals are a WARNING
+  // by default and a hard FAILURE under --leak-scan-strict (the trunk still carries residual
+  // history until task 1002219 sweeps it + task 1002217 adds the fail-closed floor gate). A
+  // secret-shape hit is ALWAYS a non-fatal warning — the R88 functional-verbatim fix is preserved.
+  if (!process.argv.includes('--no-leak-scan')) {
+    const scanFiles = staged.map((s) => ({ path: s.name.startsWith(TAR_ROOT) ? s.name.slice(TAR_ROOT.length) : s.name, content: s.buf }));
+    const scan = scanArtifactForLeaks(scanFiles);
+    const strict = process.argv.includes('--leak-scan-strict');
+    if (scan.literalHits.length) {
+      const byLit = {};
+      for (const h of scan.literalHits) byLit[h.sample] = (byLit[h.sample] || 0) + 1;
+      const summary = Object.entries(byLit).sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k}×${n}`).join(', ');
+      const files = [...new Set(scan.literalHits.map((h) => h.path))];
+      console.error('');
+      console.error(`[leak-scan] ${strict ? 'FAILED' : 'WARNING'} — ${scan.literalHits.length} prior-instance literal hit(s) across ${files.length} shipped file(s): ${summary}`);
+      for (const p of files.slice(0, 30)) console.error(`   ${p}`);
+      if (files.length > 30) console.error(`   … +${files.length - 30} more file(s)`);
+      if (strict) { console.error('[leak-scan] --leak-scan-strict: refusing to certify a leaky artifact.'); process.exit(1); }
+      console.error('[leak-scan] non-fatal for now — becomes a hard gate once the trunk sweep (task 1002219) + the fail-closed floor gate (task 1002217) land. Run --leak-scan-strict to enforce today.');
+    } else {
+      console.error('[leak-scan] ✓ no prior-instance literals in the built artifact.');
+    }
+    if (scan.secretWarnings.length) {
+      console.error(`[leak-scan] ! ${scan.secretWarnings.length} functional file(s) carry a credential SHAPE (WARNING, not fatal — confirm none is a real credential; a real one must be removed at SOURCE):`);
+      for (const w of scan.secretWarnings.slice(0, 20)) console.error(`   ${w.path}  [${w.shapes.join(', ')}]`);
+      if (scan.secretWarnings.length > 20) console.error(`   … +${scan.secretWarnings.length - 20} more`);
+    } else {
+      console.error('[leak-scan] ✓ no committed-credential shapes in functional files.');
+    }
+  }
+
   // Optional publish leg (task 2089): --publish runs `npm publish` on the built
   // tarball. DRY-RUN by default; the real, permanent publish needs --publish --live
   // (+ npm auth). Kept AFTER the artifact is written so a publish failure never
@@ -730,12 +981,20 @@ module.exports = {
   octal, tarChecksum, ustarHeader, splitUstarName, buildTar,
   // task 1824 — the consumer-core file-classification layer (pure; unit-tested)
   isDocPath, functionalSecretLeaks, processCoreFile,
+  // idea 1000543 — npm-never-installed files dropped from the pin so vendored `bongos upgrade` pin-verify matches
+  isNpmStripped, NPM_STRIPPED_BASENAMES,
   // task 2121 — agent-doc (CLAUDE.md/AGENTS.md) stub so a consumer's node_modules can't auto-load core memory
   isAgentDoc, CORE_AGENT_DOC_STUB,
   // task 1882 (ADR 0108 §2) — the real installable npm package
   CORE_DEP_NAMES, buildPackageJson, resolveLockDep, lockfileClosure, prunePackageLock,
   // task 2089 (ADR 0134) — private-by-default npm publish
   publishArgs,
+  // task 1002218 (audit H3) — build-time self-verify (reproduce pin + optional bare-install boot)
+  selfVerify, reproducePinFromTree,
+  // task 1002220 (audit H6) — post-build leak scan (prior-instance literals + functional secret shapes)
+  scanArtifactForLeaks, PRIOR_INSTANCE_LITERALS,
+  // task 1002217 (audit H4) — the fixed neutral floor injected into the doc-redaction + fail-closed gate
+  PRIOR_INSTANCE_FLOOR,
 };
 
 if (require.main === module) main();