@bongos/core 1.19.1035 → 1.19.1037

package/scripts/gds/audit-ships.js

@@ -1,319 +0,0 @@
-// Retroactive audit script for task #204.
-//
-// Loads every task shipped in the last `days` (default 2), fetches full
-// task data via the authenticated /tasks/:id API, derives diff stats from
-// git (one commit per task, found by the `<VERSION> #<id>:` subject prefix),
-// then calls src/bongos/grader.js in-process to get a score.
-//
-// Output: JSON to stdout (--json) or human-readable table (default).
-// Audit-only — does NOT write to task_grades. Pre-grader ships keep clean
-// history.
-
-const { execFileSync } = require('node:child_process');
-const grader = require('../../modules/grading/grader');
-const { cliClient } = require('./cli-lib');
-
-// EVERY git call here goes through argv, never a shell string (task 1003223 —
-// the class task 1002761 named and task 1002526 fixed in docs-entropy).
-//
-// WHAT WAS WRONG. Five call sites built shell strings. THREE of them interpolated
-// a quoted empty `--format=''`; a POSIX shell strips those quotes and git sees an
-// empty format, but the shell Node uses on Windows does not, so git receives
-// `--format=''` literally and dies with `fatal: invalid --pretty format: ''`.
-// Verified on win32 before fixing: the shell-string form throws, the argv form
-// returns " 1 file changed, 139 insertions(+)". The correct argv spelling is
-// `--format=` — empty value, no quotes, which no shell can mangle because no
-// shell is involved. The other two passed `--format='%H %s'`, which survives a
-// POSIX shell and arrives with literal quotes elsewhere; same class, same fix.
-//
-// WHY IT MATTERED MORE THAN A LOUD CRASH. The originating row asked whether the
-// try/catch around these calls "degrades to a silently empty audit — if so it is
-// the same dangerous shape after all". It did. Every catch returned null or [],
-// so on Windows the audit reported no diff stats and zero downstream impact for
-// EVERY ship, and looked like a clean run. `gitFails` below keeps the same return
-// values but says so on stderr, so an empty audit can never again be mistaken for
-// an uneventful one.
-//
-// One of the five also built a `grep | head | awk` pipeline and pinned
-// `shell: '/bin/bash'` to survive; that filtering is now done in JS, so the script
-// needs no shell at all.
-function gitLines(args, { cwd } = {}) {
-  return execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })
-    .split('\n').map((s) => s.trim()).filter(Boolean);
-}
-
-// One place that reports a git failure instead of swallowing it. Returns the
-// caller's fallback so behaviour is unchanged; only the silence is.
-function gitFails(what, err, fallback) {
-  const msg = String((err && (err.stderr || err.message)) || err).trim().split('\n')[0];
-  console.error(`audit-ships: ${what} failed — ${msg}`);
-  return fallback;
-}
-
-const DAYS = Number(process.env.AUDIT_DAYS || 2);
-
-// `git log --all` once per RUN, not once per task.
-//
-// The shell pipeline this replaced ended in `head -1`, which let git stop as soon
-// as one subject matched. The argv rewrite has no such early exit — and
-// findCommitForTask is called once per shipped task, up to 500 in a run — so a
-// naive port turned one bounded call into 500 full history walks. The commit list
-// cannot change mid-run, so it is read once and reused. Caught by the grader on
-// the first round of this task, which is a fair catch: replacing a shell pipeline
-// means inheriting the work its operators were doing, not just its output.
-let commitSubjectsCache = null;
-function commitSubjects() {
-  if (commitSubjectsCache) return commitSubjectsCache;
-  commitSubjectsCache = gitLines(['log', '--all', '--format=%H %s']);
-  return commitSubjectsCache;
-}
-
-function findCommitForTask(taskId) {
-  // Matches both subject conventions: the legacy ' #NNN:' prefix and the de-#-ed
-  // 'task NNN:' form ship.js writes since task 916 (a bare "#NNN" in commit
-  // subjects autolinked to dead GitHub issue numbers).
-  //
-  // The grep|head|awk pipeline this replaces needed `shell: '/bin/bash'` to run at
-  // all — a hard dependency on a POSIX shell at a fixed path, for filtering JS can
-  // do. Doing it here also makes the id match EXACT: the old regex was
-  // interpolated unescaped, so a task id was pasted straight into a regex.
-  const re = new RegExp(` (?:#|task )${String(taskId).replace(/[^0-9]/g, '')}:`);
-  let lines;
-  try {
-    lines = commitSubjects();
-  } catch (e) {
-    return gitFails(`finding the commit for task ${taskId}`, e, null);
-  }
-  const hit = lines.find((l) => re.test(l));
-  return hit ? (hit.split(' ')[0] || null) : null;
-}
-
-function gitDiffStatsForCommit(sha) {
-  if (!sha) return null;
-  let raw;
-  try {
-    raw = execFileSync('git', ['show', '--shortstat', '--format=', sha], { encoding: 'utf8' });
-  } catch (e) {
-    return gitFails(`reading diff stats for ${String(sha).slice(0, 8)}`, e, null);
-  }
-  // Example shortstat line: " 15 files changed, 1171 insertions(+), 65 deletions(-)"
-  const m = raw.match(/(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?/);
-  if (!m) return null;
-  return {
-    files_changed: Number(m[1] || 0),
-    lines_added: Number(m[2] || 0),
-    lines_removed: Number(m[3] || 0),
-    commit_sha: sha,
-  };
-}
-
-function filesTouchedByCommit(sha) {
-  if (!sha) return [];
-  try {
-    return gitLines(['show', '--name-only', '--format=', sha]);
-  } catch (e) {
-    return gitFails(`listing files touched by ${String(sha).slice(0, 8)}`, e, []);
-  }
-}
-
-// Downstream-impact heuristic: number of distinct commits on main reachable
-// from origin/main that landed AFTER `sha` and touched any of `files`. High
-// count = subsequent work has built on top of this change, so a "redo"
-// risks conflicts. Excludes `sha` itself.
-const NO_DOWNSTREAM = Object.freeze({
-  commits_after_touching_files: 0, distinct_files_touched: 0, sample_subjects: [],
-});
-
-function downstreamImpact(sha, files) {
-  if (!sha || files.length === 0) return { ...NO_DOWNSTREAM };
-  // The hand-rolled POSIX shell-quoting this replaces ("'" + escape + "'") was
-  // both unnecessary and wrong on Windows, where that quoting style is not the
-  // shell's. In argv form a path with spaces, quotes or a `$` is just an element.
-  const range = `${sha}..origin/main`;
-  let lines;
-  try {
-    lines = gitLines(['log', range, '--format=%H %s', '--', ...files]);
-  } catch (e) {
-    return gitFails(`reading downstream commits after ${String(sha).slice(0, 8)}`, e, { ...NO_DOWNSTREAM });
-  }
-  // Distinct downstream files touched
-  let distinct = 0;
-  try {
-    distinct = new Set(gitLines(['log', range, '--name-only', '--format=', '--', ...files])).size;
-  } catch (e) {
-    // Same shape as every other call site: the fallback comes FROM gitFails, so
-    // the value and the report can never drift apart.
-    distinct = gitFails(`reading downstream files after ${String(sha).slice(0, 8)}`, e, 0);
-  }
-  return {
-    commits_after_touching_files: lines.length,
-    distinct_files_touched: distinct,
-    sample_subjects: lines.slice(0, 3).map((l) => l.split(' ').slice(1).join(' ')),
-  };
-}
-
-async function fetchShipped(days) {
-  // Derive the public read host from branding (statusOrigin → publicOrigin), never a
-  // hardcoded founder host (was https://status.example.com — task 1002234). The
-  // recent-shipped feed is a public endpoint served on the status dashboard host.
-  const b = (() => { try { return require('../../src/branding').branding(); } catch (_) { return {}; } })();
-  const d = b.domains || {};
-  const statusBase = String(d.statusOrigin || d.publicOrigin || '').replace(/\/+$/, '');
-  const r = await fetch(
-    `${statusBase}/api/bongos/public/recent-shipped?days=${days}&limit=500`
-  );
-  const data = await r.json();
-  return data.tasks || [];
-}
-
-async function fetchFullTask(id) {
-  const api = await cliClient();
-  const r = await api.tasks.getTasksId({ id });
-  if (!r.ok) return null;
-  return r.data.task;
-}
-
-async function fetchLearning(taskId) {
-  // /api/bongos/learnings?task=<id> — we just need body_md.
-  const api = await cliClient();
-  const r = await api.learnings.getLearnings({ query: { task: taskId } });
-  if (!r.ok) return null;
-  const rows = r.data.learnings || [];
-  return rows[0] ? { body_md: rows[0].body_md } : null;
-}
-
-async function main() {
-  console.error(`Loading shipped tasks (last ${DAYS} days)...`);
-  const shipped = await fetchShipped(DAYS);
-  console.error(`Got ${shipped.length} shipped tasks. Fetching full data + diffs...`);
-
-  const results = [];
-  for (const t of shipped) {
-    const full = await fetchFullTask(t.id);
-    if (!full) {
-      console.error(`  #${t.id}: could not fetch full task — skip`);
-      continue;
-    }
-    const sha = findCommitForTask(t.id);
-    const diff = gitDiffStatsForCommit(sha);
-    const files = filesTouchedByCommit(sha);
-    const downstream = downstreamImpact(sha, files);
-    const verification = {
-      scanner_clean: true, // got shipped → passed
-      smoke_clean: true,
-      diff_stats: diff || undefined,
-    };
-    const learningRow =
-      full.kind === 'learning-capture' ? await fetchLearning(full.id) : null;
-
-    // V3.R18 (#209): the runtime grade entry switched to gradeSubagent;
-    // this retroactive #204 audit keeps using the v1 deterministic scorer
-    // because the historical signal we're measuring (process audit, not
-    // subagent quality) is what's in the v1 rubric. The function was
-    // renamed from grade() to gradeDeterministic() in the same commit.
-    const g = grader.gradeDeterministic({
-      task: full,
-      verification,
-      actualMinutes: full.actual_minutes,
-      valueSummary: full.value_summary,
-      learningRow,
-    });
-
-    // Categorize failure mode. Only meaningful when g.passed === false.
-    // - calibration-noise: only criterion under 5 is est_actual_band
-    // - quality-concern:   multiple criteria under 5
-    // - structural-risk:   downstream impact high (≥10 commits or ≥10 distinct files)
-    let category = null;
-    if (!g.passed) {
-      const lowCriteria = Object.entries(g.rubric_scores)
-        .filter(([_k, v]) => v < 5)
-        .map(([k]) => k);
-      const onlyCalibrationLow = lowCriteria.length === 1 && lowCriteria[0] === 'est_actual_band';
-      const heavyDownstream =
-        downstream.commits_after_touching_files >= 10 ||
-        downstream.distinct_files_touched >= 10;
-      if (heavyDownstream) category = 'structural-risk';
-      else if (onlyCalibrationLow) category = 'calibration-noise';
-      else category = 'quality-concern';
-    }
-
-    results.push({
-      id: full.id,
-      title: full.title,
-      version_id: full.version_id,
-      kind: full.kind,
-      discipline: full.discipline,
-      est_minutes: full.est_minutes,
-      actual_minutes: full.actual_minutes,
-      scoping_confidence: full.scoping_confidence,
-      value_summary: full.value_summary,
-      description: full.description,
-      shipped_at: t.shipped_at,
-      diff_stats: diff,
-      files_touched: files,
-      downstream,
-      grade: g,
-      category,
-    });
-  }
-
-  if (process.argv.includes('--json')) {
-    console.log(JSON.stringify(results, null, 2));
-    return;
-  }
-
-  // Human-readable summary.
-  const passed = results.filter((r) => r.grade.passed);
-  const failed = results.filter((r) => !r.grade.passed);
-  console.log(`\n=== Audit summary ===`);
-  console.log(`Total graded:  ${results.length}`);
-  console.log(`Passed:        ${passed.length}`);
-  console.log(`Failed:        ${failed.length}  (threshold ${grader.RUBRIC.v1.threshold}, v1 deterministic)`);
-  console.log('');
-
-  if (failed.length === 0) {
-    console.log('No failures. Nothing to redo.');
-    return;
-  }
-
-  console.log('=== FAILURES (sorted by score asc) ===\n');
-  failed
-    .sort((a, b) => a.grade.score - b.grade.score)
-    .forEach((r) => {
-      console.log(`#${r.id}  score ${r.grade.score}/10  [${r.kind}]  est=${r.est_minutes}m act=${r.actual_minutes}m`);
-      console.log(`  ${r.title}`);
-      const scores = r.grade.rubric_scores;
-      const parts = Object.entries(scores).map(([k, v]) => `${k}=${v}`);
-      console.log(`  ${parts.join('  ')}`);
-      console.log('');
-    });
-
-  // Score histogram for the passed set so we see what the distribution looks like.
-  const buckets = { '5.0-5.9': 0, '6.0-6.9': 0, '7.0-7.9': 0, '8.0-8.9': 0, '9.0-10': 0 };
-  for (const r of passed) {
-    const s = r.grade.score;
-    if (s < 6) buckets['5.0-5.9']++;
-    else if (s < 7) buckets['6.0-6.9']++;
-    else if (s < 8) buckets['7.0-7.9']++;
-    else if (s < 9) buckets['8.0-8.9']++;
-    else buckets['9.0-10']++;
-  }
-  console.log('=== Passed-score histogram ===');
-  for (const [k, v] of Object.entries(buckets)) console.log(`  ${k}: ${v}`);
-}
-
-// Run only as a CLI. Without this guard the file executes main() — and hits the
-// API — the instant anything requires it, which is why its git helpers had never
-// been tested: there was no way to reach them without starting an audit
-// (task 1003223).
-if (require.main === module) {
-  main().catch((err) => {
-    console.error(err);
-    process.exit(1);
-  });
-}
-
-// Exported for tests/audit_ships_git_argv.mjs — the argv-vs-shell-string fix is a
-// property of THESE functions, so the test drives them rather than re-typing an
-// equivalent git call and hoping it matches.
-module.exports = { findCommitForTask, gitDiffStatsForCommit, filesTouchedByCommit, downstreamImpact };

package/tests/audit_ships_git_argv.mjs

@@ -1,173 +0,0 @@
-// tests/audit_ships_git_argv.mjs — audit-ships' git calls must be argv, not a
-// shell string (task 1003223; the class task 1002761 named).
-//
-// THE BUG. Five calls interpolated `--format=''` into a shell string. A POSIX
-// shell strips the quotes and git sees an empty format; the shell Node uses on
-// Windows does not, so git receives `--format=''` literally and dies with
-// `fatal: invalid --pretty format: ''`. A sixth built a `grep | head | awk`
-// pipeline and pinned `shell: '/bin/bash'` to survive at all.
-//
-// WHY THIS NEEDS A TEST AND NOT JUST A FIX. Every one of those calls sat inside a
-// try/catch that returned null or []. So the Windows failure did not crash the
-// audit — it produced an audit with no diff stats and zero downstream impact for
-// every ship, which is indistinguishable from a clean run. The originating row
-// asked exactly this ("check whether the catch degrades to a silently empty audit
-// — if so it is the same dangerous shape after all"). It did.
-//
-// The static half of this file is the real net: it is the half that stays true on
-// Linux CI, where the shell-string form would have passed happily forever.
-//
-// Run:  node --test tests/audit_ships_git_argv.mjs
-
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import fs from 'node:fs';
-import path from 'node:path';
-import { execSync, execFileSync } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
-import { createRequire } from 'node:module';
-
-const require = createRequire(import.meta.url);
-const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-const FILE = path.join(ROOT, 'scripts/gds/audit-ships.js');
-const audit = require('../scripts/gds/audit-ships.js');
-const raw = fs.readFileSync(FILE, 'utf8');
-
-// Scan CODE, not prose. The file's header explains the bug by quoting the exact
-// tokens that are banned ("--format=''", "grep -E", "shell: '/bin/bash'"), so a
-// naive scan of the whole file fails on its own documentation — which is exactly
-// what happened when this test was first written. Same comments-stripped posture
-// as scripts/gds/category-advisory-guard.js.
-const src = raw
-  .replace(/\/\*[\s\S]*?\*\//g, '')          // block comments
-  .split('\n').map((l) => l.replace(/(^|\s)\/\/.*$/, '$1')).join('\n');
-
-// ---- static: the shape that broke must not come back ----------------------
-
-test('no execSync anywhere — a shell string is what mangles the quoting', () => {
-  assert.doesNotMatch(src, /\bexecSync\b/,
-    'audit-ships must spawn git through execFileSync with an argv array');
-});
-
-test("no quoted --format='' — the argv spelling is --format= with no quotes", () => {
-  // Match the SHELL form precisely: two consecutive quotes after the '='. A naive
-  // /--format='/ also matches the correct argv literal '--format=', where that
-  // quote is JS string syntax, not a shell quote — it flagged the fix as the bug.
-  assert.doesNotMatch(src, /--format=''/,
-    "a quoted empty format is the exact token git rejects on Windows");
-  assert.match(src, /'--format='/, 'the argv spelling is present');
-});
-
-test('no shell pipeline and no pinned /bin/bash', () => {
-  assert.doesNotMatch(src, /\bshell:\s*['"]/, 'no shell is needed once calls are argv');
-  for (const tool of ['grep -E', '| head', '| awk']) {
-    assert.ok(!src.includes(tool), `${tool} is a POSIX-only dependency`);
-  }
-});
-
-test('git failures are reported, not swallowed into a clean-looking audit', () => {
-  // The silent `catch (_) { return null; }` is what made the Windows breakage
-  // invisible. Every catch around a git call must go through gitFails.
-  assert.match(raw, /function gitFails\(/, 'the reporting helper exists');
-  // Scoped to catches returning an empty RESULT SET — null or [] — which is the
-  // shape that turned a git failure into a clean-looking audit. A catch returning
-  // `{}` for an unreadable branding pack is a different, legitimate default and is
-  // deliberately not swept up here.
-  const silent = src.match(/catch \(_\)\s*\{\s*return (?:null|\[\])/g) || [];
-  assert.equal(silent.length, 0,
-    `silent catch(es) returning an empty result: ${silent.join(' | ')}`);
-});
-
-// ---- behavioural: the calls actually work on THIS platform ----------------
-//
-// These run real git against this repo, which is what makes them meaningful on
-// Windows: before the fix, two of them threw here and returned empty.
-
-const git = (args) => execFileSync('git', args, { cwd: ROOT, encoding: 'utf8' }).trim();
-// A NON-MERGE commit. `git show --name-only` prints nothing for a merge by
-// default, so anchoring on HEAD makes the file-listing assertions depend on
-// whatever happened to be checked out — they failed here for that reason, not
-// because the code was wrong. --no-merges picks a commit that really touched files.
-const HEAD = git(['rev-list', '--no-merges', '-n', '1', 'HEAD']);
-
-test('the exact argv git rejected as a shell string succeeds here', () => {
-  const out = execFileSync('git', ['show', '--shortstat', '--format=', HEAD], { cwd: ROOT, encoding: 'utf8' });
-  assert.match(out, /\d+ files? changed/, `shortstat should parse, got: ${JSON.stringify(out.slice(0, 80))}`);
-});
-
-test('the shell-string form is what actually fails on this platform', () => {
-  // Not a hypothetical: assert the OLD form's behaviour so the reason for the fix
-  // is recorded in a runnable way. On POSIX it succeeds (the quotes are stripped),
-  // on Windows it throws — either way, argv is the only spelling that is correct
-  // on both, which is the whole point.
-  let threw = false;
-  try { execSync(`git show --shortstat --format='' ${HEAD}`, { cwd: ROOT, encoding: 'utf8' }); }
-  catch { threw = true; }
-  if (process.platform === 'win32') {
-    assert.equal(threw, true, 'on win32 the quoted form must be the one that fails');
-  } else {
-    assert.equal(threw, false, 'on POSIX it happens to work — which is why this hid on CI');
-  }
-});
-
-test('--name-only --format= yields real paths', () => {
-  const out = execFileSync('git', ['show', '--name-only', '--format=', HEAD], { cwd: ROOT, encoding: 'utf8' });
-  const files = out.split('\n').map((s) => s.trim()).filter(Boolean);
-  assert.ok(files.length > 0, 'HEAD touched at least one file');
-  assert.ok(files.every((f) => !f.includes("'")), 'paths come back unquoted');
-});
-
-// ---- the real functions, not equivalents ---------------------------------
-//
-// These drive audit-ships' own helpers. Before this task the file executed main()
-// on require — it hit the API the moment anything imported it — so its git helpers
-// had never been reachable from a test at all. That is the deeper reason the
-// Windows breakage went unnoticed for months.
-
-test('gitDiffStatsForCommit returns real stats on this platform', () => {
-  const stats = audit.gitDiffStatsForCommit(HEAD);
-  assert.ok(stats, 'must not be the null the Windows failure returned');
-  assert.equal(stats.commit_sha, HEAD);
-  assert.ok(stats.files_changed > 0, `files_changed should be positive, got ${stats.files_changed}`);
-});
-
-test('filesTouchedByCommit returns real paths, not the empty array of a swallowed failure', () => {
-  const files = audit.filesTouchedByCommit(HEAD);
-  assert.ok(Array.isArray(files) && files.length > 0, `expected paths, got ${JSON.stringify(files)}`);
-  assert.ok(files.every((f) => typeof f === 'string' && f.length > 0));
-});
-
-test('the commit list is read once per run, not once per task', () => {
-  // The shell pipeline this replaced ended in `head -1`, so git stopped at the
-  // first match. The argv form has no early exit, and findCommitForTask runs once
-  // per shipped task — up to 500 a run — so a naive port turns one bounded call
-  // into 500 full history walks. Caught by the grader on this task's first round.
-  //
-  // Measured rather than asserted structurally: the second lookup must not spend
-  // anywhere near the first's time, because it must not touch git at all.
-  const t0 = process.hrtime.bigint();
-  audit.findCommitForTask(999999901);
-  const cold = process.hrtime.bigint() - t0;
-
-  const t1 = process.hrtime.bigint();
-  audit.findCommitForTask(999999902);
-  const warm = process.hrtime.bigint() - t1;
-
-  assert.ok(warm * 5n < cold || warm < 2_000_000n,
-    `second lookup should reuse the cached list (cold ${cold / 1000000n}ms, warm ${warm / 1000000n}ms)`);
-});
-
-test('the helpers still answer safely for a missing sha', () => {
-  assert.equal(audit.gitDiffStatsForCommit(null), null);
-  assert.deepEqual(audit.filesTouchedByCommit(null), []);
-  assert.deepEqual(audit.downstreamImpact(null, ['x']),
-    { commits_after_touching_files: 0, distinct_files_touched: 0, sample_subjects: [] });
-});
-
-test('a path containing a space survives argv — the hand-rolled quoting is gone', () => {
-  // git log -- <pathspec> with a space used to need shell quoting; in argv it is
-  // simply one element. An unmatched pathspec returns empty, never an error.
-  const out = execFileSync('git', ['log', '-1', '--format=%H', '--', 'no such file.md'], { cwd: ROOT, encoding: 'utf8' });
-  assert.equal(out.trim(), '', 'an unmatched pathspec is empty, not a crash');
-});
-