@bongos/core 1.21.174 → 1.21.176

package/scripts/bongos/ship-grade.js

@@ -0,0 +1,426 @@
+// scripts/bongos/ship-grade.js — grader baseline/diff/fingerprint, the subagent-grader runner and the smoke runners.
+//
+// Carved out of scripts/bongos/ship.js by task 1003205 (goal 1000079, the R03
+// de-monolith plan) — a STRUCTURAL move, not a rewrite: the code below is the
+// original ship.js lines 1763-2049 unchanged. ship.js remains the facade that
+// re-exports every name and owns the CLI entry point, so the ship contract and
+// every flag are untouched, and the other scripts that import from it are too.
+
+'use strict';
+
+const crypto = require('node:crypto');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+const { cliClient } = require('./cli-lib');
+const grader = require('../../modules/grading/grader');
+// task 1003977: the ONE definition of "does this ship put copy in front of a
+// person". It lives in the lifecycle module and the grading module may not
+// require a sibling (ADR 0083), so this script — which is not a module and
+// already requires grading directly — is the seam where the two meet. Writing a
+// second predicate inside grading would let the at-ship reader lens and the
+// after-the-fact artist review disagree about what they cover, and widening the
+// set would then have to be done twice.
+const { hasCopyOrVisualWork } = require('../../modules/lifecycle/cascade.js');
+const { resolveBuilderModel } = require('./ship-memory-sync.js');
+const { argvOut, gitArgvOk, gitOk, shellOk } = require('./ship-git.js');
+const { GRADER_DIFF_CHAR_CAP, GRADER_TIMEOUT_MS, CORE_ROOT, REPO_ROOT, noteServerReward } = require('./ship-state.js');
+
+// The core's smoke suite, by absolute path from the package dir (task 1004584).
+const SMOKE_SCRIPT = path.join(CORE_ROOT, 'scripts', 'bongos', 'smoke-gds.js');
+
+// ---------- Step 1+2: local verification ----------
+
+// V3.R18 (#209) → MAS Phase 1 #395: inputs for the subagent grader. The grader
+// needs three things from the local working tree: the changed files list, the
+// full unified diff against the baseline, and (for the audit-trail signals)
+// the shortstat summary.
+//
+// task 1002422 (blocker 1000082): a builder-settable pre-task baseline. When a
+// task's artifact reached main BEFORE the ship (a release task: package-core
+// archives committed HEAD, so the version bump must merge first), merge-base(
+// origin/main, HEAD) == HEAD → an EMPTY committed diff → the --allow-empty /
+// skip-grade / Archon-only /confirm dead-end that NO sub-Archon builder can
+// finish. --base <ref> pins the pre-task commit so the grader reads the REAL diff
+// <base>...HEAD and the ship confirms via the rank-open graded path instead. Set
+// once in main() after resolveBaseRef() validates it; null on a normal ship.
+let baseOverrideSha = null;
+
+// resolveBaseRef — validate + resolve a --base <ref> to a commit sha. Injection-
+// safe: the ref reaches git as a literal argv element (execFile, no shell), never
+// a string interpolation. Pure over injected git runners so the unit test drives
+// it with fakes (no real repo). Returns { ok:true, sha } or { ok:false, reason }.
+function resolveBaseRef(rawBase, { revParse, isAncestor } = {}) {
+  const ref = String(rawBase == null ? '' : rawBase).trim();
+  if (!ref) return { ok: false, reason: 'empty ref' };
+  // Conservative allowlist (defence-in-depth even though transport is argv): no
+  // whitespace, no leading dash (git would read it as a flag), no shell metachar.
+  if (!/^[A-Za-z0-9][A-Za-z0-9._/~^-]*$/.test(ref)) {
+    return { ok: false, reason: `invalid ref syntax "${ref}"` };
+  }
+  const rp = revParse || ((r) => argvOut('git', ['rev-parse', '--verify', '--quiet', `${r}^{commit}`]));
+  const sha = rp(ref);
+  if (!sha) return { ok: false, reason: `"${ref}" is not a commit this repo knows` };
+  const anc = isAncestor || ((s) => gitArgvOk(['merge-base', '--is-ancestor', s, 'HEAD']));
+  if (!anc(sha)) {
+    return { ok: false, reason: `"${ref}" (${sha.slice(0, 9)}) is not an ancestor of HEAD — --base must be a commit your work built on top of` };
+  }
+  return { ok: true, sha };
+}
+
+// baseRefShipGuard — the SECURITY invariant behind --base. --base decouples the
+// GRADED diff (<base>...HEAD) from what the merge leg lands (the whole branch:
+// autoMerge runs `git merge <branch>`, bringing every commit ahead of main, not
+// just <base>...HEAD). So if HEAD has commits AHEAD of origin/main, honoring
+// --base would let the grader review only a SUBSET while the full branch merges —
+// an unreviewed-code path onto main. --base is therefore ONLY valid for work
+// ALREADY fully on main (its documented premise: "nothing left to merge"). Pure
+// over the ahead-count string (git rev-list --count origin/main..HEAD) so it is
+// unit-tested without a repo. '0' → allowed; anything else (or null) → refuse.
+function baseRefShipGuard(aheadCount) {
+  if (aheadCount === '0') return { ok: true };
+  const n = aheadCount == null ? 'an unknown number of' : aheadCount;
+  return { ok: false, reason: `HEAD is ${n} commit(s) ahead of origin/main — --base is only for work already merged to main` };
+}
+
+// applyBaseOverride — the SINGLE source of truth for --base validation, shared by
+// shipMain + regradeMain so the security guard can never diverge between entry
+// points. Resolves the ref, enforces the 0-ahead invariant, and sets the module-
+// level baseOverrideSha on success. Returns { ok:true, sha } | { ok:false, reason }
+// (the caller prints + exits). runGit is injectable for tests.
+function applyBaseOverride(baseRefRaw, { runGit = gitOk } = {}) {
+  const r = resolveBaseRef(baseRefRaw);
+  if (!r.ok) return { ok: false, reason: `--base ${baseRefRaw} is unusable (${r.reason})` };
+  // Freshen origin/main (best-effort; a stale ref only OVER-counts → errs to refuse).
+  runGit('git fetch origin main --quiet');
+  const guard = baseRefShipGuard(runGit('git rev-list --count origin/main..HEAD'));
+  if (!guard.ok) return { ok: false, reason: guard.reason };
+  baseOverrideSha = r.sha;
+  return { ok: true, sha: r.sha };
+}
+
+// #536: the grader's baseline is computed INDEPENDENTLY of touches[] — always
+// from merge-base, never gated on the declared scope. (Historically the
+// touches-gated drift scanner returned a null baseline for empty touches[],
+// which once blinded the grader to a fully-committed diff and hard-FAILed
+// empty-touches newcomer ships; the scanner is gone as of task 879, but the
+// grader keeping its own merge-base baseline remains the correct invariant.)
+// baseSha (task 1002422): a validated --base override wins over merge-base, so a
+// task whose commits already landed on main still grades its real diff instead of
+// seeing an empty one.
+function computeGraderBaseline(claim, baseSha = baseOverrideSha) {
+  if (baseSha) return baseSha;
+  return (
+    gitOk('git merge-base origin/main HEAD') ||
+    gitOk('git merge-base main HEAD') ||
+    (claim && claim.head_sha) ||
+    null
+  );
+}
+
+function computeChangedFiles(baseline) {
+  if (!baseline) return [];
+  const out = gitOk(`git diff --name-only ${baseline}...HEAD`);
+  return out ? out.split('\n').map((s) => s.trim()).filter(Boolean) : [];
+}
+
+// How much diff we are willing to CAPTURE. Distinct from GRADER_DIFF_CHAR_CAP, which is
+// how much we are willing to SHOW the panel: the capture has to succeed first, and a
+// capture that FAILS is indistinguishable from "nothing changed" unless we look.
+const DIFF_CAPTURE_MAX_BYTES = 512 * 1024 * 1024;
+
+function computeDiffText(baseline) {
+  if (!baseline) return '';
+  // -M to track renames; --no-color so the subagent gets clean diff lines.
+  //
+  // maxBuffer is set EXPLICITLY, and that is the whole point of this function. execSync
+  // defaults to 1 MB and gitOk swallows the resulting ENOBUFS as null, so a large change
+  // silently produced an EMPTY diff: the panel then graded a zero-character string and
+  // would have returned a verdict on nothing at all, reporting diff_chars=0 in a log line
+  // that reads like an ordinary stat rather than an error. A 1,141-file rename (task
+  // 1003705) measured 9,808,703 chars and is what surfaced it; every big change had the
+  // same hole. The GRADER_DIFF_CHAR_CAP truncation below still bounds what the subagent is
+  // asked to read — this only stops the capture itself from failing.
+  const diff = gitOk(`git diff --no-color -M ${baseline}...HEAD`, { maxBuffer: DIFF_CAPTURE_MAX_BYTES });
+  if (diff === null) {
+    // Capture failed even at that size. Fall back to the stat summary so the panel gets the
+    // SHAPE of the change and can Read any file in its post-resolve state — strictly better
+    // than an empty string, which reads as "no changes" and grades as a pass.
+    // gitOk returns null ONLY on failure; a genuinely empty diff comes back as ''.
+    const stat = gitOk(`git diff --no-color -M --stat ${baseline}...HEAD`, { maxBuffer: 64 * 1024 * 1024 });
+    return stat
+      ? `[diff too large to capture in full — stat summary only. Read any file for its post-resolve state.]\n\n${stat}`
+      : '';
+  }
+  if (!diff) return '';
+  if (diff.length > GRADER_DIFF_CHAR_CAP) {
+    const head = diff.slice(0, GRADER_DIFF_CHAR_CAP);
+    return `${head}\n\n[diff truncated — original was ${diff.length} chars, capped at ${GRADER_DIFF_CHAR_CAP}. The subagent can Read any file in its post-resolve state for the full picture.]`;
+  }
+  return diff;
+}
+
+function computeDiffShortstat(baseline) {
+  if (!baseline) return null;
+  const summary = gitOk(`git diff --shortstat ${baseline}...HEAD`);
+  if (!summary) return { files_changed: 0, lines_added: 0, lines_removed: 0 };
+  const files = /(\d+)\s+files?\s+changed/.exec(summary);
+  const ins = /(\d+)\s+insertions?\(\+\)/.exec(summary);
+  const del = /(\d+)\s+deletions?\(-\)/.exec(summary);
+  return {
+    files_changed: files ? Number(files[1]) : 0,
+    lines_added:   ins   ? Number(ins[1])   : 0,
+    lines_removed: del   ? Number(del[1])   : 0,
+  };
+}
+
+// ---------- V4.R08 — diff fingerprint + grade reuse ----------
+
+// SHA-256 of the raw diff string. Stable identifier for "has the committed
+// diff changed?" — used as the key to skip re-running the adversarial panel
+// when the diff is byte-for-byte identical to the last PASS.
+function diffHash(rawDiff) {
+  return crypto.createHash('sha256').update(rawDiff || '').digest('hex');
+}
+
+// Compute the hash of the full (untruncated) diff against baseline. Returns
+// null when baseline is unavailable, OR when the diff cannot be captured, so
+// callers can treat null as "unknown".
+//
+// SAME BUFFER BUG AS computeDiffText, AND A WORSE CONSEQUENCE. This ran gitOk at
+// execSync's default 1 MB, so a large diff came back null and the old
+// `diffHash(raw || '')` hashed the EMPTY STRING — the same fixed SHA-256 for every
+// large diff. shouldReuseGrade compares that hash against the stored grade's
+// diff_hash, so two genuinely different large diffs on one task collided and a stale
+// PASS could be reused on changed code, skipping the panel outright. That is the
+// silent-pass failure this task exists to remove, reappearing in the reuse path.
+//
+// Returning null on a failed capture is the fail-safe answer, not a cosmetic one:
+// shouldReuseGrade already refuses a falsy currentDiffHash, so "unknown" now means
+// "run the panel", where "" meant "matches every other unknown".
+function computeRawDiffHash(baseline) {
+  if (!baseline) return null;
+  const raw = gitOk(`git diff --no-color -M ${baseline}...HEAD`, { maxBuffer: DIFF_CAPTURE_MAX_BYTES });
+  if (raw === null) return null; // capture failed — never collapse that to hash('')
+  return diffHash(raw);
+}
+
+// Pure predicate: should we reuse the stored grade instead of running the
+// panel again? True when a PASS grade exists for the exact same diff hash.
+// A null storedGrade, a non-PASS grade, or a hash mismatch all return false.
+function shouldReuseGrade(storedGrade, currentDiffHash) {
+  if (!storedGrade || !storedGrade.passed) return false;
+  if (!storedGrade.diff_hash || !currentDiffHash) return false;
+  return storedGrade.diff_hash === currentDiffHash;
+}
+
+// task 1002660: task_grades has no issues column — the panel's deduped
+// issues[] survive only inside signals.workers[].issues. Rebuild them (same
+// (file, summary) dedupe as aggregatePanel) so the reuse re-POST carries the
+// substance the original PASS carried. Before this, the reused body posted
+// issues: undefined, so every reused PASS with findings read as a zero-issue
+// pass server-side and tripped the SR-9 plausibility guard into
+// manual-confirm — 0/106 successful uses, the feature defeated itself.
+function reconstructPanelIssues(signals) {
+  const out = [];
+  const seen = new Set();
+  for (const w of (Array.isArray(signals?.workers) ? signals.workers : [])) {
+    for (const it of (Array.isArray(w?.issues) ? w.issues : [])) {
+      if (!it || typeof it !== 'object') continue;
+      const key = `${it.file || ''}|${it.summary || ''}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(it);
+    }
+  }
+  return out;
+}
+
+// --- Subagent grader runner ---
+// Wraps grader.gradeSubagent so ship.js callers see a single function with
+// progress logging. MAS Phase 2.0 #410: gradeSubagent now runs the WORKER
+// PANEL (Narc + Quality; Hacker / Efficiency Monger added in 2.1/2.2) in
+// parallel. Each worker is a specialized prompt with its own verdict; the
+// panel gate is unanimous PASS — any 'fail' blocks, any 'concerns' falls
+// through to manual-confirm.
+async function runSubagentGrader({ task, valueSummary, baseline, changedFiles: precomputedChanged, priorRound = null }) {
+  // Reuse the caller's changed-files list when provided (the empty-diff guard
+  // computes it just before calling us) so we don't run `git diff --name-only`
+  // twice per ship (#513).
+  const changedFiles = precomputedChanged || computeChangedFiles(baseline);
+  const diff = computeDiffText(baseline);
+  const shortstat = computeDiffShortstat(baseline);
+  // SEC #863: feed the grader the builder's LIVE rank so the deterministic
+  // permission-path pre-pass can hard-fail a sub-Metic diff that touches the
+  // rank/permission/ship-grade/deploy machinery. Best-effort read; if it fails
+  // the pre-pass fails OPEN (the pre-push hook + post-push main audit are the
+  // fail-closed layers).
+  let builderRank;
+  try {
+    const meRes = noteServerReward(await (await cliClient()).me.getMe());
+    if (meRes.ok) builderRank = meRes.data?.builder?.rank;
+  } catch (_) { /* unknown rank → pre-pass fails open; audit + hook still gate */ }
+  const runOpts = {
+    timeoutMs: GRADER_TIMEOUT_MS,
+    cwd: process.cwd(), // grade the FEATURE worktree, not REPO_ROOT
+  };
+  // task 1002659: the real session model (transcript-derived) drives the
+  // cross-model grader pick; null → the grader assumes, as before.
+  const builderModel = resolveBuilderModel();
+  const t0 = Date.now();
+  let result;
+  // #513: a small copy/docs-only diff gets a SINGLE generalist grader shot
+  // rather than the full adversarial panel — "lighter, not skipped". The 1-shot
+  // still returns ff+cq+economy+confidence so the server gate applies; the
+  // deterministic scanner + smoke + route-rank checks ran regardless. Fixes the
+  // inefficiency of running a 4-worker panel on a +1/-1 copy tweak.
+  if (grader.isTrivialDiff({ changedFiles, diffStats: shortstat })) {
+    const totalLines = (shortstat.lines_added || 0) + (shortstat.lines_removed || 0);
+    console.log(`  subagent grader ${grader.VERSION}: TRIVIAL diff (${shortstat.files_changed} file(s), ${totalLines} line(s), copy/docs only) → single generalist shot, model=${grader.selectGraderModel({ builderModel })} — instead of the ${grader.WORKERS.DEFAULT_PANEL.length}-worker panel (#513)`);
+    console.log(`  gate: the shot must return functional_fidelity>=${grader.THRESHOLD} AND code_quality>=${grader.THRESHOLD} AND confidence>=0.4`);
+    result = await grader.gradeSubagentV3MultiShot({
+      task,
+      valueSummary,
+      diff,
+      changedFiles,
+      diffStats: shortstat,
+      shots: 1,
+      builderModel, // task 1002659: real session model → cross-model grader pick
+      runOpts,
+    });
+    // task 1002660: the single generalist shot has no Hacker or Efficiency
+    // lens — acceptable for genuinely inert copy/docs files, but never
+    // silently. The marker makes the dropped lenses queryable by /grade-audit
+    // (same pattern as implausible_pass_waived, ADR 0164).
+    result.signals = {
+      ...(result.signals || {}),
+      trivial_path: true,
+      lenses_dropped: ['hacker', 'efficiency'],
+    };
+  } else {
+    // task 1003977: the advisory Reader rides along on copy-bearing diffs only.
+    // Where the predicate says no, `panel` is byte-identical to DEFAULT_PANEL
+    // and the ship costs exactly what it costs today.
+    const copyBearing = hasCopyOrVisualWork(task, changedFiles);
+    const panel = grader.WORKERS.panelFor({ copyBearing });
+    console.log(`  subagent grader ${grader.VERSION}: panel of ${panel.length} workers (${panel.join(', ')}), model=${grader.selectGraderModel({ builderModel })} (builder ${builderModel || `${grader.ASSUMED_BUILDER_MODEL} assumed`}), changed_files=${changedFiles.length}, diff_chars=${diff.length}, per-worker timeout=${GRADER_TIMEOUT_MS}ms`);
+    console.log(`  gate: every GATING worker must return verdict='pass'; 'concerns' → manual-confirm fall-through; 'fail' blocks`);
+    console.log(copyBearing
+      ? `  copy lens: ON — this task is copy-bearing, so the advisory 'reader' reads the changed strings against docs/voice.md. It is NON-GATING: nothing it reports, and no error or timeout it hits, can stop this ship.`
+      : `  copy lens: off — not a copy-bearing task (no copy-proposal source, not artist-discipline, module not in COPY_VISUAL_MODULES, no page file under modules/*/public/ changed), so the panel is the usual four.`);
+    if (priorRound) {
+      console.log(`  delta re-grade (task 1002662): prior round's findings + builder response ride along, fenced; answered findings can only re-veto with NEW evidence`);
+    }
+    result = await grader.gradeSubagent({
+      task,
+      valueSummary,
+      diff,
+      changedFiles,
+      diffStats: shortstat, // V3.R92 #353: feed diff size into the economy axis
+      builderRank,          // SEC #863: drives the permission-path pre-pass
+      builderModel,         // task 1002659: real session model → cross-model grader pick
+      priorRound,           // task 1002662: delta-aware re-grade context (panel path only)
+      workers: panel,       // task 1003977: DEFAULT_PANEL, plus the advisory reader when copy-bearing
+      runOpts,
+    });
+    // A SKIPPED LENS IS RECORDED, NEVER SILENT. Without this a grade with no
+    // copy_score is indistinguishable from one where the copy was read and
+    // found clean — a missing score would read as a passing one. Same pattern
+    // and same reason as the trivial path's `lenses_dropped` above.
+    if (!copyBearing) {
+      result.signals = {
+        ...(result.signals || {}),
+        reader_skipped: 'not_copy_bearing',
+      };
+    }
+  }
+  const wallMs = Date.now() - t0;
+  // Decorate signals with the local shortstat — useful in audit-trail.
+  // graded_head_sha (task 1002662): record WHAT commit this grade judged, so a
+  // later --regrade can compute the interim fix diff (prior graded HEAD → now)
+  // for the delta-aware prior-round payload.
+  result.signals = {
+    ...(result.signals || {}),
+    diff_stats: shortstat,
+    wall_ms: wallMs,
+    changed_files_count: changedFiles.length,
+    graded_head_sha: gitOk('git rev-parse HEAD') || null,
+  };
+  return result;
+}
+
+// task 1004011: the no-artifact twin of runSubagentGrader. Same station in the
+// ship, different subject — there is no diff, so the thing graded is the builder's
+// CLAIM that none was needed (modules/grading/grader-no-artifact.js, ADR 0320).
+//
+// WHY a separate runner rather than a flag on the one above: every input that one
+// assembles is diff-derived (changed files, the unified diff, the shortstat, the
+// generated-provenance check, the deterministic route-rank and permission-path
+// pre-passes over the changed files). On an empty diff each of those is an empty
+// set, and an empty set fed to a check that reports "no hits" is a check that
+// passes vacuously — the exact failure mode task 1002646's Guard 1b exists to
+// catch server-side. Threading `if (empty)` through all of them would be a wider
+// change with more places to get that wrong than a sibling that never computes
+// them at all.
+async function runNoArtifactGrader({ task, valueSummary, notes, runOpts: extraOpts = {} }) {
+  const builderModel = resolveBuilderModel();
+  const graderModel = grader.selectGraderModel({ builderModel });
+  const runOpts = {
+    timeoutMs: GRADER_TIMEOUT_MS,
+    cwd: process.cwd(), // grade from the FEATURE worktree — the Read tool is the only evidence check here
+    ...extraOpts,
+  };
+  console.log(`  subagent grader ${grader.VERSION}: NO-ARTIFACT grade (empty committed diff) — one generalist shot over the handoff notes + value summary, model=${graderModel}`);
+  console.log(`  subject: the CLAIM that nothing was needed, judged against the task description — not a diff, and not the absence of one`);
+  console.log(`  gate: unchanged — functional_fidelity>=${grader.THRESHOLD} AND code_quality>=${grader.THRESHOLD} AND confidence>=0.4, re-asked for this species (is it genuinely artifact-free / is the evidence checkable)`);
+  const t0 = Date.now();
+  const result = await grader.gradeNoArtifact({ task, valueSummary, notes, builderModel, runOpts });
+  result.signals = {
+    ...(result.signals || {}),
+    wall_ms: Date.now() - t0,
+    changed_files_count: 0,
+    graded_head_sha: gitOk('git rev-parse HEAD') || null,
+  };
+  return result;
+}
+
+function runSmoke() {
+  // The script is addressed from CORE_ROOT, never cwd-relative: in a vendored
+  // instance it lives under node_modules/@bongos/core, not under the instance
+  // checkout, and the relative form died MODULE_NOT_FOUND after a clean merge —
+  // which then reset main and stranded the task at confirmed (task 1004584).
+  // cwd stays the instance checkout so instance config resolves as before; the
+  // JSON.stringify quoting survives spaces in either path.
+  const okPms = shellOk(`"${process.execPath}" ${JSON.stringify(SMOKE_SCRIPT)}`, { cwd: REPO_ROOT });
+  return { ran: true, clean: okPms };
+}
+
+// task 1002660: the async twin of runSmoke for the PRE-RESOLVE gate only. The
+// sync runSmoke blocks the event loop (execSync), which would freeze the
+// overlapped grade-panel promise mid-flight; awaiting this one keeps the loop
+// free so the panel's subprocess callbacks progress while smoke runs. The
+// post-merge smoke call keeps the sync form (nothing overlaps there).
+function runSmokeAsync() {
+  return new Promise((resolve) => {
+    const child = spawn(process.execPath, [SMOKE_SCRIPT], {
+      cwd: REPO_ROOT, stdio: 'inherit', windowsHide: true,
+    });
+    child.on('error', () => resolve({ ran: true, clean: false }));
+    child.on('close', (code) => resolve({ ran: true, clean: code === 0 }));
+  });
+}
+
+module.exports = {
+  applyBaseOverride,
+  baseRefShipGuard,
+  computeChangedFiles,
+  computeGraderBaseline,
+  computeRawDiffHash,
+  diffHash,
+  reconstructPanelIssues,
+  resolveBaseRef,
+  runNoArtifactGrader,
+  runSmoke,
+  runSmokeAsync,
+  runSubagentGrader,
+  shouldReuseGrade,
+};