@bongos/core 1.21.174 → 1.21.176

package/scripts/bongos/mirror-lineage-check.js

@@ -0,0 +1,343 @@
+#!/usr/bin/env node
+'use strict';
+
+// scripts/bongos/mirror-lineage-check.js — the C9 PROOF: private⇄public lineage
+// consistency + a no-leak check (GDS-V4 · V4.R73 · task 1212, implementing the
+// ADR 0099 "Consequences" recipe).
+//
+// C9 (Cloud Bongos ⇄ Example decoupling) publishes a redacted, 6-month-
+// delayed public mirror of the private build repo (task 1211, scripts/public-
+// mirror-export.js). This check asserts the two lineages can't silently diverge:
+//
+//   1. NO-LEAK / NO-TOPOLOGY — redact the whole publishable tree exactly as the
+//      export does (mirror-redact + secret-scrub, via the SHARED redactBlob) and
+//      assert ZERO residual instance literals, public IPs, or secrets survive.
+//      Same detector the export gates on — "one detector, two consumers."
+//   2. MIGRATION NUMBERING — the set of core migration numbers (bare NNN_*.sql and
+//      namespaced core_NNN_*.sql alike) that reach the mirror is IDENTICAL to the
+//      private set (redaction never renames a file), so the claim-time next-number
+//      allocator computes the SAME next migration on both sides. A dropped or
+//      renumbered migration = divergence = fail.
+//   3. ENUM VALUES — for every publishable migration, the CHECK (... IN (...))
+//      value-lists are byte-identical before and after redaction. Redaction may
+//      touch a domain in a comment; it must NEVER alter a schema enum value.
+//
+// TWO MODES:
+//   • default (simulation) — proves the invariant against the CURRENT tree. This
+//     is the live C9 acceptance while the repo is younger than the 180-day publish
+//     lag (the export is a deliberate no-op until then, so there is no mirror yet).
+//   • --mirror <dir> — the live check once the pipeline is publishing: reads the
+//     mirror's .mirror-provenance.json, asserts source_commit is an ancestor of
+//     private main, then diffs the mirror's migrations/enums against that commit
+//     and no-leak-scans the mirror's actual files.
+//
+// Exit 0 = consistent + clean; 1 = divergence or a residual leak; 2 = bad usage.
+// Pure helpers (parseEnumChecks / migrationNumbersFromPaths / numberingDivergence
+// / enumDrift) are exported for tests/mirror_lineage_proof.mjs.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+const { isPublishable } = require(path.join(REPO_ROOT, 'scripts/bongos/publish-manifest.js'));
+const {
+  buildRedactionRules, collectSensitiveLiterals, redactText, findLeaks,
+} = require(path.join(REPO_ROOT, 'scripts/bongos/mirror-redact.js'));
+const { isBinary } = require(path.join(REPO_ROOT, 'scripts/public-mirror-export.js'));
+const { scrubSecrets, collectEnvSecrets, SECRET_PATTERNS } = require(path.join(REPO_ROOT, 'modules/security/secret-scrub.js'));
+const { nextFree } = require(path.join(REPO_ROOT, 'scripts/bongos/next-number.js'));
+const { migrationNumberOf } = require(path.join(REPO_ROOT, 'scripts/bongos/migration-namespace.js'));
+
+// Known secret SHAPES as NON-GLOBAL clones. We scan each file with .test(), and a
+// `/g` regex carries lastIndex state across .test() calls — reused in a loop over
+// files that makes the result order-dependent (the exact footgun that leaves the
+// export's own findSecretLeak evaluation-order-sensitive; task 1212 flagged it,
+// see docs/reviews/gds-v4-c9-lineage-proof.md). Stateless clones keep this proof
+// deterministic regardless of file order.
+const KNOWN_SECRET_SHAPES = SECRET_PATTERNS.map((re) => new RegExp(re.source, re.flags.replace('g', '')));
+
+// ---- pure helpers (exported for tests) ------------------------------------
+
+// The CORE migration number for a repo-relative path, or null if it is not one.
+//
+// Core migrations sit DIRECTLY under migrations/ as either the legacy bare NNN_slug.sql
+// or the namespaced core_NNN_slug.sql every NEW core migration must use (ADR 0108 §5).
+// The prefix namespaces the FILENAME (so a core stem can't collide with an instance/
+// module one across repos), NOT the number line — core_188 continues the bare 001–187
+// sequence as ONE sequence. Hence the shared migrationNumberOf, the SAME parser the
+// claim-time allocator (next-number.js) counts from: this proof and the allocator can
+// never disagree about which numbers are taken. (Task 1002527 — the bare-regex version
+// was blind to core_188+, leaving BOTH halves equally blind and the gate falsely green.)
+//
+// Module migrations (modules/<key>/migrations/) and instance SEEDS
+// (migrations/instance/, ADR 0105) are numbered independently and excluded from the
+// publishable mirror; the allocator only reads the top-level core dir, so only core
+// numbering can diverge.
+function coreMigrationNumber(repoRelPath) {
+  const s = String(repoRelPath);
+  if (!s.startsWith('migrations/')) return null; // the migrations dir …
+  // … and ONLY its top level. Counting migrations/instance/NNN_*.sql put every instance
+  // seed in the PRIVATE set but not the MIRROR set (they are unpublishable, task 2140) —
+  // a false private⇄mirror divergence that red-lined the unit gate (task 2152). A path
+  // with a further slash is a subdir, never a core migration.
+  const rest = s.slice('migrations/'.length);
+  if (rest.includes('/')) return null;
+  return migrationNumberOf(rest);
+}
+
+// The set of CORE migration numbers among a list of repo-relative paths.
+function migrationNumbersFromPaths(paths) {
+  const out = new Set();
+  for (const p of paths || []) {
+    const n = coreMigrationNumber(p);
+    if (n != null) out.add(n);
+  }
+  return out;
+}
+
+// Extract the enum value-lists from a migration's SQL: every `IN ( '...' , ... )`
+// clause's quoted literals. Returns a deterministic (sorted) structure so it can
+// be deep-compared before/after redaction. Non-enum `IN (...)` (a subquery with
+// no string literals) contributes nothing. Robust to multi-line lists.
+function parseEnumChecks(sql) {
+  const lists = [];
+  if (typeof sql !== 'string') return lists;
+  const inRe = /\bIN\s*\(([^)]*)\)/gi;
+  let m;
+  while ((m = inRe.exec(sql)) !== null) {
+    const vals = m[1].match(/'((?:[^']|'')*)'/g) || [];
+    if (vals.length) lists.push([...vals].sort());
+  }
+  lists.sort((a, b) => (a.join('\u0000') < b.join('\u0000') ? -1 : a.join('\u0000') > b.join('\u0000') ? 1 : 0));
+  return lists;
+}
+
+// Compare two migration-number sets. { missing, extra, consistent }.
+function numberingDivergence(privateSet, mirrorSet) {
+  const priv = privateSet instanceof Set ? privateSet : new Set(privateSet);
+  const mir = mirrorSet instanceof Set ? mirrorSet : new Set(mirrorSet);
+  const missing = [...priv].filter((n) => !mir.has(n)).sort((a, b) => a - b);
+  const extra = [...mir].filter((n) => !priv.has(n)).sort((a, b) => a - b);
+  return { missing, extra, consistent: missing.length === 0 && extra.length === 0 };
+}
+
+// Which migrations had their enum value-lists changed by redaction. files:
+// [{ path, original, redacted }] → [{ path, before, after }].
+function enumDrift(files) {
+  const drift = [];
+  for (const f of files || []) {
+    const before = parseEnumChecks(f.original);
+    const after = parseEnumChecks(f.redacted);
+    if (JSON.stringify(before) !== JSON.stringify(after)) drift.push({ path: f.path, before, after });
+  }
+  return drift;
+}
+
+// ---- git plumbing (private repo = REPO_ROOT) ------------------------------
+function git(args) {
+  return execFileSync('git', ['-C', REPO_ROOT, ...args], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+}
+
+function buildRedactionContext() {
+  let branding = {};
+  try {
+    branding = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'config/branding.json'), 'utf8'));
+  } catch { /* vanilla instance — only the public-IP floor + secret scrub apply */ }
+  const extra = {
+    owners: String(process.env.OWNER_ALIASES || '').split(',').map((s) => s.trim()).filter(Boolean),
+    literals: process.env.DROPLET_IP ? [process.env.DROPLET_IP] : [],
+  };
+  return {
+    rules: buildRedactionRules(branding, extra),
+    sensitiveLiterals: collectSensitiveLiterals(branding, extra),
+    envSecrets: collectEnvSecrets(),
+  };
+}
+
+// Redact + scan ONE file the way the export publishes it, then check it for
+// residual leaks — DETERMINISTICALLY (no dependency on the export's evaluation-
+// order-sensitive findSecretLeak). Text files are redacted (instance literals +
+// IPs) then secret-scrubbed — those exact bytes are what the mirror would carry;
+// binary files ride verbatim but are still latin1-scanned. Returns
+// { redactedText|null, leaks:[{label,sample?}] }.
+function scanFile(buf, ctx) {
+  const { rules, sensitiveLiterals, envSecrets } = ctx;
+  let scanText;
+  let redactedText = null;
+  if (isBinary(buf)) {
+    scanText = buf.toString('latin1');
+  } else {
+    redactedText = scrubSecrets(redactText(buf.toString('utf8'), rules), envSecrets);
+    scanText = redactedText;
+  }
+  // Instance literals + public IPs (the topology/no-instance guarantee — findLeaks
+  // resets its own IP-regex lastIndex, so it is order-safe).
+  const leaks = findLeaks(scanText, sensitiveLiterals).map((l) => ({ label: l.label, sample: l.sample }));
+  // Any KNOWN secret shape surviving the scrub (stateless clones — see above).
+  for (const re of KNOWN_SECRET_SHAPES) {
+    if (re.test(scanText)) { leaks.push({ label: 'secret-shape' }); break; }
+  }
+  return { redactedText, leaks };
+}
+
+// The shared assertion core: given the publishable file set (as {path, buf}),
+// run the three checks. Returns a report.
+function proveLineage({ publishableFiles, privateMigrationNumbers, ctx }) {
+  const leaks = [];
+  const migFiles = [];
+  const mirrorNums = new Set();
+
+  for (const f of publishableFiles) {
+    const { redactedText, leaks: fileLeaks } = scanFile(f.buf, ctx);
+    for (const l of fileLeaks) leaks.push({ path: f.path, label: l.label, sample: l.sample });
+    // Same predicate as the private half (migrationNumbersFromPaths) — the two sets are
+    // only comparable if both are built by the identical parser.
+    const migNum = coreMigrationNumber(f.path);
+    if (migNum != null) {
+      migFiles.push({ path: f.path, original: f.buf.toString('utf8'), redacted: redactedText != null ? redactedText : f.buf.toString('utf8') });
+      mirrorNums.add(migNum);
+    }
+  }
+
+  const numbering = numberingDivergence(privateMigrationNumbers, mirrorNums);
+  const drift = enumDrift(migFiles);
+  return {
+    leaks,
+    numbering,
+    nextPrivate: nextFree(privateMigrationNumbers),
+    nextMirror: nextFree(mirrorNums),
+    drift,
+    counts: { publishable: publishableFiles.length, migrations: migFiles.length },
+  };
+}
+
+// ---- mode: simulation (default) -------------------------------------------
+function runSimulation() {
+  const ctx = buildRedactionContext();
+  const tracked = git(['ls-files']).split('\n').map((s) => s.trim()).filter(Boolean);
+  const publishablePaths = tracked.filter(isPublishable);
+  const privateMigrationNumbers = migrationNumbersFromPaths(tracked);
+
+  const publishableFiles = [];
+  for (const p of publishablePaths) {
+    try {
+      publishableFiles.push({ path: p, buf: fs.readFileSync(path.join(REPO_ROOT, p)) });
+    } catch (err) {
+      // A tracked-but-unreadable file is itself a failure to surface.
+      publishableFiles.push({ path: p, buf: Buffer.from('') });
+      console.error(`[c9] could not read ${p}: ${err.message}`);
+    }
+  }
+  return proveLineage({ publishableFiles, privateMigrationNumbers, ctx });
+}
+
+// ---- mode: live (--mirror <dir>) ------------------------------------------
+function runAgainstMirror(mirrorDir) {
+  const ctx = buildRedactionContext();
+  const provPath = path.join(mirrorDir, '.mirror-provenance.json');
+  const prov = JSON.parse(fs.readFileSync(provPath, 'utf8'));
+  const sourceCommit = prov.source_commit;
+  if (!sourceCommit) throw new Error('.mirror-provenance.json has no source_commit');
+
+  // Lineage anchor: source_commit MUST be an ancestor of private main (HEAD here).
+  let ancestor = false;
+  try {
+    execFileSync('git', ['-C', REPO_ROOT, 'merge-base', '--is-ancestor', sourceCommit, 'HEAD'], { stdio: 'ignore' });
+    ancestor = true;
+  } catch { ancestor = false; }
+
+  // Private publishable migration numbers AT the source commit (the mirror is a
+  // redacted copy of exactly that commit's publishable tree).
+  const srcPaths = git(['ls-tree', '-r', '--name-only', sourceCommit]).split('\n').map((s) => s.trim()).filter(Boolean);
+  const privateMigrationNumbers = migrationNumbersFromPaths(srcPaths.filter(isPublishable));
+
+  // The mirror's ACTUAL files (already redacted) — scan them as-is for leaks +
+  // read migration enums straight off disk. We compare mirror migration numbers +
+  // enums to the source commit.
+  const walk = (dir, base = '') => {
+    const out = [];
+    for (const name of fs.readdirSync(dir)) {
+      if (name === '.git') continue;
+      const abs = path.join(dir, name);
+      const rel = base ? `${base}/${name}` : name;
+      if (fs.statSync(abs).isDirectory()) out.push(...walk(abs, rel));
+      else out.push({ path: rel, buf: fs.readFileSync(abs) });
+    }
+    return out;
+  };
+  const mirrorFiles = walk(mirrorDir).filter((f) => f.path !== '.mirror-provenance.json');
+
+  // The mirror is already redacted, so "redact again" is a no-op; redactBlob still
+  // gives us the leak scan. proveLineage compares mirror migration numbers to the
+  // source commit's publishable set and checks enum stability (idempotent here).
+  const report = proveLineage({ publishableFiles: mirrorFiles, privateMigrationNumbers, ctx });
+  report.ancestor = ancestor;
+  report.sourceCommit = sourceCommit;
+  return report;
+}
+
+// ---- report + exit --------------------------------------------------------
+function printReport(report, { mode }) {
+  const log = console.log;
+  log(`\nC9 lineage proof — ${mode}`);
+  log(`  publishable files: ${report.counts.publishable}   core migrations: ${report.counts.migrations}`);
+  if (report.sourceCommit) {
+    log(`  source_commit: ${String(report.sourceCommit).slice(0, 12)}  ancestor-of-main: ${report.ancestor ? 'YES' : 'NO'}`);
+  }
+
+  const ok = { leak: report.leaks.length === 0, numbering: report.numbering.consistent, enum: report.drift.length === 0, ancestor: report.ancestor !== false };
+
+  log('');
+  log(`  [${ok.leak ? '✓' : '✗'}] no-leak / no-topology   (${report.leaks.length} residual leak(s))`);
+  log(`  [${ok.numbering ? '✓' : '✗'}] migration numbering     (next: private=${report.nextPrivate} mirror=${report.nextMirror}; missing=[${report.numbering.missing.join(',')}] extra=[${report.numbering.extra.join(',')}])`);
+  log(`  [${ok.enum ? '✓' : '✗'}] enum values preserved   (${report.drift.length} migration(s) drifted)`);
+  if (report.sourceCommit) log(`  [${ok.ancestor ? '✓' : '✗'}] lineage anchor          (source_commit ancestor of private main)`);
+
+  if (!ok.leak) {
+    log('\n  Residual leaks (fix mirror-redact.js rules or publish-manifest.js exclusions):');
+    for (const l of report.leaks.slice(0, 30)) {
+      const shown = l.label === 'secret-shape' ? '' : (l.sample ? ` (“${String(l.sample).slice(0, 40)}”)` : '');
+      log(`    ${l.path}: ${l.label}${shown}`);
+    }
+    if (report.leaks.length > 30) log(`    … +${report.leaks.length - 30} more`);
+  }
+  if (!ok.numbering) log(`\n  Migration numbering DIVERGED — a migration was dropped/renamed on the way to the mirror.`);
+  if (!ok.enum) {
+    log('\n  Enum values were altered by redaction (they must be identical private⇄public):');
+    for (const d of report.drift.slice(0, 10)) log(`    ${d.path}`);
+  }
+
+  const pass = ok.leak && ok.numbering && ok.enum && ok.ancestor;
+  log(`\n${pass ? '✓ C9 lineage proof PASSED' : '✗ C9 lineage proof FAILED'} — private⇄public numbering + enums consistent, mirror clean.`);
+  return pass;
+}
+
+function main() {
+  const mIdx = process.argv.indexOf('--mirror');
+  const mirrorDir = mIdx >= 0 ? process.argv[mIdx + 1] : null;
+  try {
+    if (mirrorDir) {
+      const report = runAgainstMirror(path.resolve(mirrorDir));
+      process.exit(printReport(report, { mode: `live (mirror: ${mirrorDir})` }) ? 0 : 1);
+    }
+    const report = runSimulation();
+    process.exit(printReport(report, { mode: 'simulation (current tree — the dormant-repo acceptance)' }) ? 0 : 1);
+  } catch (err) {
+    console.error(`✖ C9 lineage proof error: ${err.message}`);
+    process.exit(1);
+  }
+}
+
+if (require.main === module) main();
+
+module.exports = {
+  coreMigrationNumber,
+  migrationNumbersFromPaths,
+  parseEnumChecks,
+  numberingDivergence,
+  enumDrift,
+  proveLineage,
+  buildRedactionContext,
+  runSimulation,
+};