@bongos/core 1.19.318 → 1.19.320

package/scripts/gds/ship-grade.js

@@ -0,0 +1,319 @@
+// scripts/gds/ship-grade.js — grader baseline/diff/fingerprint, the subagent-grader runner and the smoke runners.
+//
+// Carved out of scripts/gds/ship.js by task 1003205 (goal 1000079, the R03
+// de-monolith plan) — a STRUCTURAL move, not a rewrite: the code below is the
+// original ship.js lines 1763-2049 unchanged. ship.js remains the facade that
+// re-exports every name and owns the CLI entry point, so the ship contract and
+// every flag are untouched, and the other scripts that import from it are too.
+
+'use strict';
+
+const crypto = require('node:crypto');
+const { spawn } = require('node:child_process');
+const { cliClient } = require('./cli-lib');
+const grader = require('../../modules/grading/grader');
+const { resolveBuilderModel } = require('./ship-memory-sync.js');
+const { argvOut, gitArgvOk, gitOk, shellOk } = require('./ship-git.js');
+const { GRADER_DIFF_CHAR_CAP, GRADER_TIMEOUT_MS, REPO_ROOT, noteServerReward } = require('./ship-state.js');
+
+// ---------- Step 1+2: local verification ----------
+
+// V3.R18 (#209) → MAS Phase 1 #395: inputs for the subagent grader. The grader
+// needs three things from the local working tree: the changed files list, the
+// full unified diff against the baseline, and (for the audit-trail signals)
+// the shortstat summary.
+//
+// task 1002422 (blocker 1000082): a builder-settable pre-task baseline. When a
+// task's artifact reached main BEFORE the ship (a release task: package-core
+// archives committed HEAD, so the version bump must merge first), merge-base(
+// origin/main, HEAD) == HEAD → an EMPTY committed diff → the --allow-empty /
+// skip-grade / Archon-only /confirm dead-end that NO sub-Archon builder can
+// finish. --base <ref> pins the pre-task commit so the grader reads the REAL diff
+// <base>...HEAD and the ship confirms via the rank-open graded path instead. Set
+// once in main() after resolveBaseRef() validates it; null on a normal ship.
+let baseOverrideSha = null;
+
+// resolveBaseRef — validate + resolve a --base <ref> to a commit sha. Injection-
+// safe: the ref reaches git as a literal argv element (execFile, no shell), never
+// a string interpolation. Pure over injected git runners so the unit test drives
+// it with fakes (no real repo). Returns { ok:true, sha } or { ok:false, reason }.
+function resolveBaseRef(rawBase, { revParse, isAncestor } = {}) {
+  const ref = String(rawBase == null ? '' : rawBase).trim();
+  if (!ref) return { ok: false, reason: 'empty ref' };
+  // Conservative allowlist (defence-in-depth even though transport is argv): no
+  // whitespace, no leading dash (git would read it as a flag), no shell metachar.
+  if (!/^[A-Za-z0-9][A-Za-z0-9._/~^-]*$/.test(ref)) {
+    return { ok: false, reason: `invalid ref syntax "${ref}"` };
+  }
+  const rp = revParse || ((r) => argvOut('git', ['rev-parse', '--verify', '--quiet', `${r}^{commit}`]));
+  const sha = rp(ref);
+  if (!sha) return { ok: false, reason: `"${ref}" is not a commit this repo knows` };
+  const anc = isAncestor || ((s) => gitArgvOk(['merge-base', '--is-ancestor', s, 'HEAD']));
+  if (!anc(sha)) {
+    return { ok: false, reason: `"${ref}" (${sha.slice(0, 9)}) is not an ancestor of HEAD — --base must be a commit your work built on top of` };
+  }
+  return { ok: true, sha };
+}
+
+// baseRefShipGuard — the SECURITY invariant behind --base. --base decouples the
+// GRADED diff (<base>...HEAD) from what the merge leg lands (the whole branch:
+// autoMerge runs `git merge <branch>`, bringing every commit ahead of main, not
+// just <base>...HEAD). So if HEAD has commits AHEAD of origin/main, honoring
+// --base would let the grader review only a SUBSET while the full branch merges —
+// an unreviewed-code path onto main. --base is therefore ONLY valid for work
+// ALREADY fully on main (its documented premise: "nothing left to merge"). Pure
+// over the ahead-count string (git rev-list --count origin/main..HEAD) so it is
+// unit-tested without a repo. '0' → allowed; anything else (or null) → refuse.
+function baseRefShipGuard(aheadCount) {
+  if (aheadCount === '0') return { ok: true };
+  const n = aheadCount == null ? 'an unknown number of' : aheadCount;
+  return { ok: false, reason: `HEAD is ${n} commit(s) ahead of origin/main — --base is only for work already merged to main` };
+}
+
+// applyBaseOverride — the SINGLE source of truth for --base validation, shared by
+// shipMain + regradeMain so the security guard can never diverge between entry
+// points. Resolves the ref, enforces the 0-ahead invariant, and sets the module-
+// level baseOverrideSha on success. Returns { ok:true, sha } | { ok:false, reason }
+// (the caller prints + exits). runGit is injectable for tests.
+function applyBaseOverride(baseRefRaw, { runGit = gitOk } = {}) {
+  const r = resolveBaseRef(baseRefRaw);
+  if (!r.ok) return { ok: false, reason: `--base ${baseRefRaw} is unusable (${r.reason})` };
+  // Freshen origin/main (best-effort; a stale ref only OVER-counts → errs to refuse).
+  runGit('git fetch origin main --quiet');
+  const guard = baseRefShipGuard(runGit('git rev-list --count origin/main..HEAD'));
+  if (!guard.ok) return { ok: false, reason: guard.reason };
+  baseOverrideSha = r.sha;
+  return { ok: true, sha: r.sha };
+}
+
+// #536: the grader's baseline is computed INDEPENDENTLY of touches[] — always
+// from merge-base, never gated on the declared scope. (Historically the
+// touches-gated drift scanner returned a null baseline for empty touches[],
+// which once blinded the grader to a fully-committed diff and hard-FAILed
+// empty-touches newcomer ships; the scanner is gone as of task 879, but the
+// grader keeping its own merge-base baseline remains the correct invariant.)
+// baseSha (task 1002422): a validated --base override wins over merge-base, so a
+// task whose commits already landed on main still grades its real diff instead of
+// seeing an empty one.
+function computeGraderBaseline(claim, baseSha = baseOverrideSha) {
+  if (baseSha) return baseSha;
+  return (
+    gitOk('git merge-base origin/main HEAD') ||
+    gitOk('git merge-base main HEAD') ||
+    (claim && claim.head_sha) ||
+    null
+  );
+}
+
+function computeChangedFiles(baseline) {
+  if (!baseline) return [];
+  const out = gitOk(`git diff --name-only ${baseline}...HEAD`);
+  return out ? out.split('\n').map((s) => s.trim()).filter(Boolean) : [];
+}
+
+function computeDiffText(baseline) {
+  if (!baseline) return '';
+  // -M to track renames; --no-color so the subagent gets clean diff lines.
+  const diff = gitOk(`git diff --no-color -M ${baseline}...HEAD`);
+  if (!diff) return '';
+  if (diff.length > GRADER_DIFF_CHAR_CAP) {
+    const head = diff.slice(0, GRADER_DIFF_CHAR_CAP);
+    return `${head}\n\n[diff truncated — original was ${diff.length} chars, capped at ${GRADER_DIFF_CHAR_CAP}. The subagent can Read any file in its post-resolve state for the full picture.]`;
+  }
+  return diff;
+}
+
+function computeDiffShortstat(baseline) {
+  if (!baseline) return null;
+  const summary = gitOk(`git diff --shortstat ${baseline}...HEAD`);
+  if (!summary) return { files_changed: 0, lines_added: 0, lines_removed: 0 };
+  const files = /(\d+)\s+files?\s+changed/.exec(summary);
+  const ins = /(\d+)\s+insertions?\(\+\)/.exec(summary);
+  const del = /(\d+)\s+deletions?\(-\)/.exec(summary);
+  return {
+    files_changed: files ? Number(files[1]) : 0,
+    lines_added:   ins   ? Number(ins[1])   : 0,
+    lines_removed: del   ? Number(del[1])   : 0,
+  };
+}
+
+// ---------- V4.R08 — diff fingerprint + grade reuse ----------
+
+// SHA-256 of the raw diff string. Stable identifier for "has the committed
+// diff changed?" — used as the key to skip re-running the adversarial panel
+// when the diff is byte-for-byte identical to the last PASS.
+function diffHash(rawDiff) {
+  return crypto.createHash('sha256').update(rawDiff || '').digest('hex');
+}
+
+// Compute the hash of the full (untruncated) diff against baseline. Returns
+// null when baseline is unavailable so callers can treat null as "unknown".
+function computeRawDiffHash(baseline) {
+  if (!baseline) return null;
+  const raw = gitOk(`git diff --no-color -M ${baseline}...HEAD`);
+  return diffHash(raw || '');
+}
+
+// Pure predicate: should we reuse the stored grade instead of running the
+// panel again? True when a PASS grade exists for the exact same diff hash.
+// A null storedGrade, a non-PASS grade, or a hash mismatch all return false.
+function shouldReuseGrade(storedGrade, currentDiffHash) {
+  if (!storedGrade || !storedGrade.passed) return false;
+  if (!storedGrade.diff_hash || !currentDiffHash) return false;
+  return storedGrade.diff_hash === currentDiffHash;
+}
+
+// task 1002660: task_grades has no issues column — the panel's deduped
+// issues[] survive only inside signals.workers[].issues. Rebuild them (same
+// (file, summary) dedupe as aggregatePanel) so the reuse re-POST carries the
+// substance the original PASS carried. Before this, the reused body posted
+// issues: undefined, so every reused PASS with findings read as a zero-issue
+// pass server-side and tripped the SR-9 plausibility guard into
+// manual-confirm — 0/106 successful uses, the feature defeated itself.
+function reconstructPanelIssues(signals) {
+  const out = [];
+  const seen = new Set();
+  for (const w of (Array.isArray(signals?.workers) ? signals.workers : [])) {
+    for (const it of (Array.isArray(w?.issues) ? w.issues : [])) {
+      if (!it || typeof it !== 'object') continue;
+      const key = `${it.file || ''}|${it.summary || ''}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(it);
+    }
+  }
+  return out;
+}
+
+// --- Subagent grader runner ---
+// Wraps grader.gradeSubagent so ship.js callers see a single function with
+// progress logging. MAS Phase 2.0 #410: gradeSubagent now runs the WORKER
+// PANEL (Narc + Quality; Hacker / Efficiency Monger added in 2.1/2.2) in
+// parallel. Each worker is a specialized prompt with its own verdict; the
+// panel gate is unanimous PASS — any 'fail' blocks, any 'concerns' falls
+// through to manual-confirm.
+async function runSubagentGrader({ task, valueSummary, baseline, changedFiles: precomputedChanged, priorRound = null }) {
+  // Reuse the caller's changed-files list when provided (the empty-diff guard
+  // computes it just before calling us) so we don't run `git diff --name-only`
+  // twice per ship (#513).
+  const changedFiles = precomputedChanged || computeChangedFiles(baseline);
+  const diff = computeDiffText(baseline);
+  const shortstat = computeDiffShortstat(baseline);
+  // SEC #863: feed the grader the builder's LIVE rank so the deterministic
+  // permission-path pre-pass can hard-fail a sub-Metic diff that touches the
+  // rank/permission/ship-grade/deploy machinery. Best-effort read; if it fails
+  // the pre-pass fails OPEN (the pre-push hook + post-push main audit are the
+  // fail-closed layers).
+  let builderRank;
+  try {
+    const meRes = noteServerReward(await (await cliClient()).me.getMe());
+    if (meRes.ok) builderRank = meRes.data?.builder?.rank;
+  } catch (_) { /* unknown rank → pre-pass fails open; audit + hook still gate */ }
+  const runOpts = {
+    timeoutMs: GRADER_TIMEOUT_MS,
+    cwd: process.cwd(), // grade the FEATURE worktree, not REPO_ROOT
+  };
+  // task 1002659: the real session model (transcript-derived) drives the
+  // cross-model grader pick; null → the grader assumes, as before.
+  const builderModel = resolveBuilderModel();
+  const t0 = Date.now();
+  let result;
+  // #513: a small copy/docs-only diff gets a SINGLE generalist grader shot
+  // rather than the full adversarial panel — "lighter, not skipped". The 1-shot
+  // still returns ff+cq+economy+confidence so the server gate applies; the
+  // deterministic scanner + smoke + route-rank checks ran regardless. Fixes the
+  // inefficiency of running a 4-worker panel on a +1/-1 copy tweak.
+  if (grader.isTrivialDiff({ changedFiles, diffStats: shortstat })) {
+    const totalLines = (shortstat.lines_added || 0) + (shortstat.lines_removed || 0);
+    console.log(`  subagent grader ${grader.VERSION}: TRIVIAL diff (${shortstat.files_changed} file(s), ${totalLines} line(s), copy/docs only) → single generalist shot, model=${grader.selectGraderModel({ builderModel })} — instead of the ${grader.WORKERS.DEFAULT_PANEL.length}-worker panel (#513)`);
+    console.log(`  gate: the shot must return functional_fidelity>=${grader.THRESHOLD} AND code_quality>=${grader.THRESHOLD} AND confidence>=0.4`);
+    result = await grader.gradeSubagentV3MultiShot({
+      task,
+      valueSummary,
+      diff,
+      changedFiles,
+      diffStats: shortstat,
+      shots: 1,
+      builderModel, // task 1002659: real session model → cross-model grader pick
+      runOpts,
+    });
+    // task 1002660: the single generalist shot has no Hacker or Efficiency
+    // lens — acceptable for genuinely inert copy/docs files, but never
+    // silently. The marker makes the dropped lenses queryable by /grade-audit
+    // (same pattern as implausible_pass_waived, ADR 0164).
+    result.signals = {
+      ...(result.signals || {}),
+      trivial_path: true,
+      lenses_dropped: ['hacker', 'efficiency'],
+    };
+  } else {
+    const panel = grader.WORKERS.DEFAULT_PANEL;
+    console.log(`  subagent grader ${grader.VERSION}: panel of ${panel.length} workers (${panel.join(', ')}), model=${grader.selectGraderModel({ builderModel })} (builder ${builderModel || `${grader.ASSUMED_BUILDER_MODEL} assumed`}), changed_files=${changedFiles.length}, diff_chars=${diff.length}, per-worker timeout=${GRADER_TIMEOUT_MS}ms`);
+    console.log(`  gate: every worker must return verdict='pass'; 'concerns' → manual-confirm fall-through; 'fail' blocks`);
+    if (priorRound) {
+      console.log(`  delta re-grade (task 1002662): prior round's findings + builder response ride along, fenced; answered findings can only re-veto with NEW evidence`);
+    }
+    result = await grader.gradeSubagent({
+      task,
+      valueSummary,
+      diff,
+      changedFiles,
+      diffStats: shortstat, // V3.R92 #353: feed diff size into the economy axis
+      builderRank,          // SEC #863: drives the permission-path pre-pass
+      builderModel,         // task 1002659: real session model → cross-model grader pick
+      priorRound,           // task 1002662: delta-aware re-grade context (panel path only)
+      runOpts,
+    });
+  }
+  const wallMs = Date.now() - t0;
+  // Decorate signals with the local shortstat — useful in audit-trail.
+  // graded_head_sha (task 1002662): record WHAT commit this grade judged, so a
+  // later --regrade can compute the interim fix diff (prior graded HEAD → now)
+  // for the delta-aware prior-round payload.
+  result.signals = {
+    ...(result.signals || {}),
+    diff_stats: shortstat,
+    wall_ms: wallMs,
+    changed_files_count: changedFiles.length,
+    graded_head_sha: gitOk('git rev-parse HEAD') || null,
+  };
+  return result;
+}
+
+function runSmoke() {
+  // `cwd: REPO_ROOT` + a relative script path is robust to spaces or other
+  // shell metacharacters that may appear in the absolute repo path on some
+  // machines.
+  const okPms = shellOk(`"${process.execPath}" scripts/gds/smoke-gds.js`, { cwd: REPO_ROOT });
+  return { ran: true, clean: okPms };
+}
+
+// task 1002660: the async twin of runSmoke for the PRE-RESOLVE gate only. The
+// sync runSmoke blocks the event loop (execSync), which would freeze the
+// overlapped grade-panel promise mid-flight; awaiting this one keeps the loop
+// free so the panel's subprocess callbacks progress while smoke runs. The
+// post-merge smoke call keeps the sync form (nothing overlaps there).
+function runSmokeAsync() {
+  return new Promise((resolve) => {
+    const child = spawn(process.execPath, ['scripts/gds/smoke-gds.js'], {
+      cwd: REPO_ROOT, stdio: 'inherit', windowsHide: true,
+    });
+    child.on('error', () => resolve({ ran: true, clean: false }));
+    child.on('close', (code) => resolve({ ran: true, clean: code === 0 }));
+  });
+}
+
+module.exports = {
+  applyBaseOverride,
+  baseRefShipGuard,
+  computeChangedFiles,
+  computeGraderBaseline,
+  computeRawDiffHash,
+  diffHash,
+  reconstructPanelIssues,
+  resolveBaseRef,
+  runSmoke,
+  runSmokeAsync,
+  runSubagentGrader,
+  shouldReuseGrade,
+};

package/scripts/gds/ship-io.js

@@ -0,0 +1,100 @@
+// scripts/gds/ship-io.js — API-error normalization and the small terminal effects (bell, post-land outcome).
+//
+// Carved out of scripts/gds/ship.js by task 1003205 (goal 1000079, the R03
+// de-monolith plan) — a STRUCTURAL move, not a rewrite: the code below is the
+// original ship.js lines 2928-3007 unchanged. ship.js remains the facade that
+// re-exports every name and owns the CLI entry point, so the ship contract and
+// every flag are untouched, and the other scripts that import from it are too.
+
+'use strict';
+
+const os = require('node:os');
+const { playSound } = require('./ship-state.js');
+
+function playShipBell() {
+  let shipWav = 'ship';
+  try {
+    const prefsPath = require('path').join(os.homedir(), '.config', 'otb', 'sound-prefs.json');
+    const raw = require('fs').readFileSync(prefsPath, 'utf8');
+    const parsed = JSON.parse(raw);
+    const ev = parsed && parsed.event_sounds && parsed.event_sounds['task-shipped'];
+    if (typeof ev === 'string') shipWav = ev;
+  } catch (_) { /* fall through to default */ }
+  playSound(shipWav);
+}
+
+// task 1002809: normalize an API error body across BOTH wire shapes — the live
+// ADR 0116 (R01) envelope `{ error: { code, message } }` and the legacy flat
+// `{ error: 'code' }`. Same normalization as claim.js:formatClaimFailure and
+// cli-lib:isBoxScope; pure + exported for tests/ship_post_land_verify.mjs.
+function normalizeApiError(data) {
+  const errRaw = data ? data.error : undefined;
+  const isEnvelope = errRaw !== null && typeof errRaw === 'object';
+  return {
+    code: isEnvelope ? errRaw.code : errRaw,
+    message: isEnvelope ? errRaw.message : undefined,
+    // task 1002789: `details` is where the envelope carries the structured
+    // payload the flat shape used to put at the TOP level (claimWorktree,
+    // step/reason, floors). A site that reads `data.<field>` directly is
+    // reading a field that moved. Fall back to the body so a legacy flat
+    // response still resolves.
+    details: (isEnvelope && errRaw.details && typeof errRaw.details === 'object')
+      ? errRaw.details
+      : (data && typeof data === 'object' ? data : {}),
+  };
+}
+
+// task 1002789: the ONE renderer for a failed API call on the ship path.
+// Every generic bail site used to hand the raw body to console.error (or
+// interpolate `data.error` into a template, which stringifies the ADR 0116
+// envelope object to the literal "[object Object]"). Both hide a fully
+// diagnosable server error: console.error's util.inspect stops at depth 2, so
+// the `details` that names WHICH git step failed prints as "[Object]".
+// Collapse the body to one readable line instead, whichever shape it is.
+// Pure + exported for tests/ship_error_shape.mjs.
+function apiErrorLine(data, status) {
+  const { code, message, details } = normalizeApiError(data);
+  const head = code
+    ? (message && message !== code ? `${code}: ${message}` : code)
+    : (message || (status ? `HTTP ${status}` : 'unknown error'));
+  // Only the small scalar fields — a details blob can carry a whole bundle.
+  const extras = Object.entries(details || {})
+    .filter(([k, v]) => k !== 'code' && k !== 'message' && k !== 'error'
+      && (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean'))
+    .map(([k, v]) => `${k}: ${String(v).replace(/\s+/g, ' ').trim()}`);
+  const line = extras.length ? `${head} (${extras.join('; ')})` : head;
+  return status ? `${line} [HTTP ${status}]` : line;
+}
+
+// task 1002809: classify a FAILED post-land POST /tasks/<id>/ship. The merge
+// already succeeded, so the server may simply have landed the task first
+// (reconcile-on-read — task 1049 — or the ~5-min reconciler racing this call).
+// The old check compared `data.error === 'task_not_confirmed'` — dead against
+// the envelope shape, so the already-landed SUCCESS rendered as a failure with
+// an "[object Object]" reason (the HANDOFF-apex-ui "400 after a successful
+// land" gotcha). liveStatus is the task's status from a verify GET (null when
+// that GET itself failed): 'shipped' is authoritative success whatever the
+// error said; task_not_confirmed with no live status keeps the task-1049
+// benefit of the doubt; any KNOWN non-shipped status stays a loud failure.
+// Pure + exported for tests/ship_post_land_verify.mjs.
+function postLandShipOutcome({ code, liveStatus }) {
+  if (liveStatus === 'shipped') {
+    return { alreadyShipped: true, why: 'the server landed it first — the failed ship call lost that race, nothing to fix' };
+  }
+  if (liveStatus == null && code === 'task_not_confirmed') {
+    return { alreadyShipped: true, why: 'the server reconciled it from the merged PR' };
+  }
+  return { alreadyShipped: false };
+}
+
+// Step 3 of the lifecycle: confirmed → shipped. Merge the branch into main,
+// deploy, flip the task to shipped, and fire the Discord broadcast (only when
+// the change actually deploys — #94 / ADR 0032; best-effort). Side-effecting;
+// the early returns are control flow only.
+
+module.exports = {
+  apiErrorLine,
+  normalizeApiError,
+  playShipBell,
+  postLandShipOutcome,
+};