@bongos/core 1.19.1036 → 1.19.1037

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/.bongos-core.json +43 -48
  2. package/.claude/skills/grade-recover/SKILL.md +1 -1
  3. package/docs/adr/0302-the-third-identical-failure-stops-the-session-and-asks.md +3 -2
  4. package/docs/adr/0329-the-rendered-fit-check-is-a-kit-floor-and-an-advisory-ship-check-row.md +2 -2
  5. package/docs/adr/README.md +1 -1
  6. package/docs/module-api-changelog.md +2 -0
  7. package/modules/economy/credits.js +1 -1
  8. package/modules/grading/grader-config.js +3 -12
  9. package/modules/grading/grader-no-artifact.js +227 -2
  10. package/modules/grading/grader-prompt.js +1 -1
  11. package/modules/grading/grader-rubric.json +1 -86
  12. package/modules/grading/grader-score.js +1 -169
  13. package/modules/grading/grader-subagent.js +1 -51
  14. package/modules/grading/grader.js +26 -211
  15. package/modules/lifecycle/db-ship.js +3 -3
  16. package/package-lock.json +2 -2
  17. package/package.json +1 -1
  18. package/release-notes.json +10 -0
  19. package/scripts/gds/claim.js +2 -1
  20. package/scripts/gds/grade-smoke.js +3 -3
  21. package/scripts/gds/render-check.js +1 -1
  22. package/scripts/gds/ship-check.js +1 -1
  23. package/scripts/gds/ship-flow.js +1 -1
  24. package/scripts/gds/ship-grade-steps.js +2 -2
  25. package/scripts/gds/visual-review-nudge.js +2 -1
  26. package/src/module-api.js +1 -1
  27. package/tests/adr_0162_gate_citation.mjs +167 -0
  28. package/tests/grader.mjs +48 -126
  29. package/tests/grader_unbacked_score.mjs +1 -1
  30. package/tests/regrade_actual_minutes.mjs +18 -17
  31. package/tests/render_check.mjs +1 -1
  32. package/tests/skill_grade_audit.mjs +3 -3
  33. package/tests/skill_grader_health.mjs +2 -2
  34. package/scripts/gds/audit-ships.js +0 -319
  35. package/tests/audit_ships_git_argv.mjs +0 -173
package/tests/grader.mjs CHANGED
@@ -1,8 +1,8 @@
1
1
  // tests/grader.mjs
2
2
  //
3
- // Grader tests. Covers both:
4
- // - the v1 deterministic process-audit grader (kept for audit-ships.js)
5
- // - the v2 subagent quality grader (the actual credits gate from V3.R18 #209)
3
+ // Grader tests: the subagent quality grader (the credits gate since V3.R18 #209).
4
+ // The v1 deterministic process-audit grader was retired in task 1002842; the
5
+ // export-surface test below pins that it stays gone.
6
6
  //
7
7
  // The v2 subagent itself is a real-network call, so the tests stop at the
8
8
  // validator + prompt-builder boundary; the network-touching end-to-end check
@@ -48,127 +48,6 @@ async function testAsync(name, fn) {
48
48
  }
49
49
  }
50
50
 
51
- // ========================================================================
52
- // v1 — deterministic grader (legacy, used only by audit-ships.js)
53
- // ========================================================================
54
-
55
- test('v1 scoreEstActualBand: ratio inside [0.5, 2.0] → 10', () => {
56
- assert.equal(grader.scoreEstActualBand(60, 60), 10);
57
- assert.equal(grader.scoreEstActualBand(30, 60), 10);
58
- assert.equal(grader.scoreEstActualBand(120, 60), 10);
59
- });
60
-
61
- test('v1 scoreEstActualBand: ratio outside band drops linearly', () => {
62
- assert.equal(grader.scoreEstActualBand(18, 60), 5);
63
- assert.equal(grader.scoreEstActualBand(360, 60), 5);
64
- });
65
-
66
- test('v1 scoreEstActualBand: extreme ratios → 0', () => {
67
- assert.equal(grader.scoreEstActualBand(6, 60), 0);
68
- assert.equal(grader.scoreEstActualBand(600, 60), 0);
69
- });
70
-
71
- test('v1 scoreEstActualBand: missing est → 6, missing actual → 5', () => {
72
- assert.equal(grader.scoreEstActualBand(60, null), 6);
73
- assert.equal(grader.scoreEstActualBand(null, 60), 5);
74
- });
75
-
76
- test('v1 scoreValueSummary: maps length + verb', () => {
77
- assert.equal(grader.scoreValueSummary(''), 0);
78
- assert.equal(grader.scoreValueSummary(null), 0);
79
- assert.equal(grader.scoreValueSummary('short note'), 5);
80
- assert.equal(grader.scoreValueSummary('a moderately worded summary that lands above forty characters'), 8);
81
- const s = 'wire the auto-grader into the completed→confirmed transition so every task gets a quality mark before its credits are paid out';
82
- assert.ok(s.length > 120);
83
- assert.equal(grader.scoreValueSummary(s), 10);
84
- });
85
-
86
- test('v1 scoreDiffHygiene: phantom ship → 0, normal → 10, too-broad → 5', () => {
87
- assert.equal(grader.scoreDiffHygiene({ files_changed: 0, lines_added: 0, lines_removed: 0 }), 0);
88
- assert.equal(grader.scoreDiffHygiene({ files_changed: 5, lines_added: 200, lines_removed: 30 }), 10);
89
- assert.equal(grader.scoreDiffHygiene({ files_changed: 80, lines_added: 1000, lines_removed: 200 }), 5);
90
- });
91
-
92
- test('v1 scoreScopingSignal: maps each level', () => {
93
- assert.equal(grader.scoreScopingSignal(0), 2);
94
- assert.equal(grader.scoreScopingSignal(1), 4);
95
- assert.equal(grader.scoreScopingSignal(2), 6);
96
- assert.equal(grader.scoreScopingSignal(3), 7);
97
- assert.equal(grader.scoreScopingSignal(4), 8);
98
- assert.equal(grader.scoreScopingSignal(5), 10);
99
- assert.equal(grader.scoreScopingSignal(null), 6);
100
- });
101
-
102
- test('v1 scoreLocNetNegative', () => {
103
- assert.equal(grader.scoreLocNetNegative({ lines_added: 20, lines_removed: 100 }), 10);
104
- assert.equal(grader.scoreLocNetNegative({ lines_added: 100, lines_removed: 20 }), 0);
105
- });
106
-
107
- test('v1 scoreLearningRow', () => {
108
- assert.equal(grader.scoreLearningRow({ body_md: 'x'.repeat(200) }), 10);
109
- assert.equal(grader.scoreLearningRow(null), 0);
110
- });
111
-
112
- test('v1 scoreArtRubricAvg clamps + nulls', () => {
113
- assert.equal(grader.scoreArtRubricAvg(8.7), 8.7);
114
- assert.equal(grader.scoreArtRubricAvg(15), 10);
115
- assert.equal(grader.scoreArtRubricAvg(-1), 0);
116
- assert.equal(grader.scoreArtRubricAvg(NaN), null);
117
- });
118
-
119
- test('v1 blendWeightsForKind: feature renorms to 1.0', () => {
120
- const w = grader.blendWeightsForKind('feature');
121
- const total = Object.values(w).reduce((a, b) => a + b, 0);
122
- assert.ok(Math.abs(total - 1.0) < 1e-6, `total=${total}, expected 1.0`);
123
- });
124
-
125
- test('v1 blendWeightsForKind: cleanup replaces diff_hygiene', () => {
126
- const w = grader.blendWeightsForKind('cleanup');
127
- assert.ok('loc_net_negative' in w);
128
- assert.ok(!('diff_hygiene' in w));
129
- });
130
-
131
- test('v1 gradeDeterministic: well-shipped feature passes', () => {
132
- const result = grader.gradeDeterministic({
133
- task: { id: 99, kind: 'feature', est_minutes: 240, actual_minutes: 200, scoping_confidence: 4 },
134
- verification: {
135
- scanner_clean: true, smoke_clean: true,
136
- diff_stats: { files_changed: 10, lines_added: 400, lines_removed: 50 },
137
- },
138
- actualMinutes: 200,
139
- valueSummary: 'wire the auto-grader into ship.js so every task gets a quality mark before credits are paid; gates the completed→confirmed transition',
140
- });
141
- assert.ok(result.score >= 9.0, `score=${result.score}`);
142
- assert.equal(result.passed, true);
143
- assert.equal(result.grader_kind, 'auto');
144
- assert.equal(result.grader_version, grader.RUBRIC.v1.version);
145
- assert.equal(result.threshold, grader.RUBRIC.v1.threshold);
146
- });
147
-
148
- test('v1 gradeDeterministic: phantom ship scores low', () => {
149
- const result = grader.gradeDeterministic({
150
- task: { id: 100, kind: 'feature', est_minutes: 240, actual_minutes: 30, scoping_confidence: 2 },
151
- verification: {
152
- scanner_clean: true, smoke_clean: true,
153
- diff_stats: { files_changed: 0, lines_added: 0, lines_removed: 0 },
154
- },
155
- actualMinutes: 30,
156
- valueSummary: '',
157
- });
158
- assert.ok(result.score < 5.0, `score=${result.score}`);
159
- assert.equal(result.passed, false);
160
- });
161
-
162
- test('v1 gradeDeterministic: notes_md says deprecated', () => {
163
- const result = grader.gradeDeterministic({
164
- task: { id: 106, kind: 'feature', est_minutes: 60, actual_minutes: 60, scoping_confidence: 4 },
165
- verification: { scanner_clean: true, smoke_clean: true, diff_stats: { files_changed: 3, lines_added: 50, lines_removed: 5 } },
166
- actualMinutes: 60,
167
- valueSummary: 'a reasonable shipped change',
168
- });
169
- assert.match(result.notes_md, /v1.*deprecated/i);
170
- });
171
-
172
51
  // ========================================================================
173
52
  // v2 — subagent grader (the credits gate as of V3.R18 #209)
174
53
  // ========================================================================
@@ -181,17 +60,29 @@ test('v4 exports the expected surface (current panel-based gate)', () => {
181
60
  'buildPrompt', 'selectGraderModel', 'validateAndScore', 'aggregateShots',
182
61
  'runSubagent', 'runOneShot', 'gradeSubagentV3MultiShot',
183
62
  // constants
184
- 'THRESHOLD', 'VERSION', 'DEFAULT_GRADER_MODEL', 'DEFAULT_SHOTS', 'ASSUMED_BUILDER_MODEL',
63
+ 'THRESHOLD', 'VERSION', 'DEFAULT_GRADER_MODEL', 'ASSUMED_BUILDER_MODEL',
185
64
  // utilities
186
65
  'extractJson', 'extractAssistantResult', 'extractCostUsd', 'RUBRIC',
187
66
  ];
188
67
  for (const name of required) {
189
68
  assert.ok(name in grader, `missing export: ${name}`);
190
69
  }
70
+ // task 1002842: the v1 generation is retired, not merely unused.
71
+ for (const gone of ['gradeDeterministic', 'scoreEstActualBand', 'scoreValueSummary', 'scoreDiffHygiene',
72
+ 'scoreScannerSmoke', 'scoreScopingSignal', 'scoreLocNetNegative', 'scoreLearningRow', 'scoreArtRubricAvg',
73
+ 'blendWeightsForKind']) {
74
+ assert.equal(gone in grader, false, `retired v1 export came back: ${gone}`);
75
+ }
76
+ assert.equal('v1' in grader.RUBRIC, false, 'the v1 rubric block is retired');
77
+ assert.equal('v3' in grader.RUBRIC, false, 'the v3 rubric block is retired');
78
+ // The moved plumbing is one implementation, reachable from both homes.
79
+ const noArtifact = require('../modules/grading/grader-no-artifact.js');
80
+ assert.equal(grader.aggregateShots, noArtifact.aggregateShots);
81
+ assert.equal(grader.runOneShot, noArtifact.runOneShot);
191
82
  assert.ok(typeof grader.THRESHOLD === 'number');
192
83
  assert.ok(grader.THRESHOLD >= 5.0 && grader.THRESHOLD <= 10.0);
193
84
  assert.ok(grader.VERSION.startsWith('v4'), `expected v4.*, got ${grader.VERSION}`);
194
- assert.equal(grader.DEFAULT_SHOTS, 3); // kept for v3 multi-shot diagnostic path
85
+ assert.equal('DEFAULT_SHOTS' in grader, false, 'the v3 3-shot default is retired (task 1002842)');
195
86
  assert.equal(grader.DEFAULT_GRADER_MODEL, 'claude-sonnet-5'); // task 1002659: Claude 5 family pin
196
87
  assert.equal(grader.ASSUMED_BUILDER_MODEL, 'claude-fable-5');
197
88
  assert.equal(grader.FALLTHROUGH_CONFIDENCE, 0.3);
@@ -2316,6 +2207,37 @@ test('isTrivialDiff: empty changed-files or zero-line diff → false', () => {
2316
2207
  );
2317
2208
  });
2318
2209
 
2210
+ // ---------- task 1002842: the kept trivial-diff fast path still works ----------
2211
+ //
2212
+ // The v3 generation is retired, but its fast path is kept (moving it onto a v4
2213
+ // panel needs a new reviewer type — an owner decision). These pin that what was
2214
+ // kept works: ONE generalist shot by default, the generalist prompt, and the
2215
+ // moved aggregateShots collapse — driven through the runShot seam, no model spawned.
2216
+
2217
+ await testAsync('fast path: one generalist shot by default, aggregated into a gradeable result', async () => {
2218
+ const calls = [];
2219
+ const r = await grader.gradeSubagentV3MultiShot({
2220
+ task: { id: 1002842, title: 't', description: 'd', kind: 'cleanup' },
2221
+ valueSummary: 'v', diff: '+ docs line', changedFiles: ['docs/x.md'], diffStats: { files_changed: 1, lines_added: 1, lines_removed: 0 },
2222
+ builderModel: 'claude-fable-5', runOpts: {},
2223
+ runShot: async (args) => { calls.push(args); return mkPassShot(8, 8, 0.9, args.shotIndex); },
2224
+ });
2225
+ assert.equal(calls.length, 1, 'the fast path runs exactly one shot unless told otherwise');
2226
+ assert.match(calls[0].prompt, /default-fail|DEFAULT/i, 'it is the generalist default-fail prompt');
2227
+ assert.equal(r.passed, true);
2228
+ assert.equal(r.rubric_scores.functional_fidelity, 8);
2229
+ assert.equal(r.rubric_scores.code_quality, 8);
2230
+ assert.equal(r.signals.confidence, 0.9, 'the server gate reads a real confidence');
2231
+ });
2232
+
2233
+ await testAsync('fast path: a failing shot still fails the grade (not loosened by the move)', async () => {
2234
+ const r = await grader.gradeSubagentV3MultiShot({
2235
+ task: { id: 1 }, valueSummary: 'v', diff: '+ x', changedFiles: ['docs/x.md'], diffStats: {},
2236
+ runOpts: {}, runShot: async (args) => mkFailShot(4, 8, 0.9, args.shotIndex),
2237
+ });
2238
+ assert.equal(r.passed, false);
2239
+ });
2240
+
2319
2241
  // ---------- Summary ----------
2320
2242
 
2321
2243
  console.log('');
@@ -14,7 +14,7 @@
14
14
  // server's ff gate failed the ship — a PASS verdict inside a FAIL grade;
15
15
  // 3. the card had no branch for it, so it printed the ordinary FAIL block.
16
16
  // The constraint the task set: close it WITHOUT letting an unbacked low score
17
- // pass and WITHOUT adding a gate (ADR 0162). Every case below that has a low
17
+ // pass and WITHOUT adding a gate (the task's own scope: it corrects what the grade SAYS). Every case below that has a low
18
18
  // score also asserts the server still refuses it.
19
19
  //
20
20
  // Run: node tests/grader_unbacked_score.mjs
@@ -32,7 +32,7 @@
32
32
  import { strict as assert } from 'node:assert';
33
33
  import { test } from 'node:test';
34
34
  import { createRequire } from 'node:module';
35
- import { readFileSync } from 'node:fs';
35
+ import { readFileSync, readdirSync } from 'node:fs';
36
36
 
37
37
  const require = createRequire(import.meta.url);
38
38
  const { buildPrompt } = require('../modules/grading/grader-prompt.js');
@@ -78,23 +78,24 @@ test('a task with no actual_minutes at all builds the same prompt', () => {
78
78
  assert.equal(withFigure, without, 'actual_minutes changes nothing about what the panel reads');
79
79
  });
80
80
 
81
- test('scoreEstActualBand is reachable only from the retroactive audit', () => {
82
- // The one scorer that DOES read the figure. It lives in gradeDeterministic,
83
- // which the grader module's own comment calls the legacy v1 entry — its single
84
- // non-test caller is scripts/gds/audit-ships.js, a process audit run by hand
85
- // over already-shipped tasks, not a gate anything passes through. A new caller
86
- // is the same risk as wiring the figure into the prompt.
81
+ test('no grader entry reads actual_minutes — the one scorer that did is retired', () => {
82
+ // scoreEstActualBand was the ONE scorer that read the figure. It lived in the v1
83
+ // gradeDeterministic, whose only non-test caller was scripts/gds/audit-ships.js;
84
+ // task 1002842 retired all three. So the stale-figure risk this file guards is now
85
+ // closed at the source, and this pins it: nothing in the grading module may
86
+ // accept or read the figure, and the retired entry may not come back.
87
87
  const grader = require('../modules/grading/grader.js');
88
- assert.equal(typeof grader.gradeDeterministic, 'function', 'the legacy entry still exists');
89
- assert.equal(grader.gradeSubagent.length > 0, true, 'and the live entry is a different function');
88
+ assert.equal(grader.gradeDeterministic, undefined, 'the retired v1 entry stays gone');
89
+ assert.equal(grader.scoreEstActualBand, undefined, 'and so does its minutes scorer');
90
+ assert.equal(typeof grader.gradeSubagent, 'function', 'the live entry is still here');
90
91
 
91
- // gradeSubagent / gradeSubagentV3MultiShot take a single options object, so the
92
- // guard that matters is which KEYS they destructure. Neither may take minutes.
93
- const src = readFileSync(new URL('../modules/grading/grader.js', import.meta.url), 'utf8');
94
- for (const fn of ['async function gradeSubagent(', 'async function gradeSubagentV3MultiShot(']) {
95
- const at = src.indexOf(fn);
96
- assert.ok(at !== -1, `${fn} is still the entry point this pins`);
97
- const sig = src.slice(at, src.indexOf(')', at));
98
- assert.ok(!/actualMinutes/.test(sig), `${fn} does not accept actualMinutes`);
92
+ // Every grading source, not just grader.js: a new reader anywhere in the module
93
+ // is the same risk as wiring the figure into the prompt.
94
+ const dir = new URL('../modules/grading/', import.meta.url);
95
+ const files = readdirSync(dir, { recursive: true }).filter((f) => String(f).endsWith('.js'));
96
+ assert.ok(files.length > 5, 'the grading module was found');
97
+ for (const f of files) {
98
+ const src = readFileSync(new URL(String(f), dir), 'utf8');
99
+ assert.doesNotMatch(src, /actualMinutes|actual_minutes/, `${f} must not read actual_minutes`);
99
100
  }
100
101
  });
@@ -1,6 +1,6 @@
1
1
  // tests/render_check.mjs — the rendered fit check (task 1004203, ADR 0329): which pages a changed path
2
2
  // reaches, which report rows it keeps, and that /ship-check shows it as a signal that never flips its exit
3
- // (ADR 0162). DB-free and browser-free; the in-page measurement is pinned in tests/ui_design_kit_fit.mjs.
3
+ // (ADR 0329, decision 3). DB-free and browser-free; the in-page measurement is pinned in tests/ui_design_kit_fit.mjs.
4
4
  import assert from 'node:assert/strict';
5
5
  import { test } from 'node:test';
6
6
  import fs from 'node:fs';
@@ -2,8 +2,8 @@
2
2
  // accountability sweep from the grader-hardening evaluation §6 (task 1002657):
3
3
  // the override ledger, dropped blocker/major findings on passing grades (with
4
4
  // the advisory-Narc attribution filter), the false-pass spot-check, and the
5
- // outage roll-up handoff — describing and queueing ONLY (ADR 0162/0158: it
6
- // never confirms a task or flips status).
5
+ // outage roll-up handoff — describing and queueing ONLY (ADR 0158 §2: it
6
+ // never confirms a task or flips status; ADR 0162: it adds no approval step to a ship).
7
7
  import assert from 'node:assert/strict';
8
8
  import { test } from 'node:test';
9
9
  import { readFileSync, existsSync } from 'node:fs';
@@ -45,6 +45,6 @@ test('/grade-audit exists, covers the sweep legs, and stays describe-and-queue o
45
45
 
46
46
  // The hard constraint.
47
47
  assert.match(md, /never confirm/i, 'states it never confirms');
48
- assert.match(md, /ADR 0162/, 'anchors the no-new-gates constraint');
48
+ assert.match(md, /ADR 0162/, 'anchors the no-approval-step-on-a-ship constraint');
49
49
  assert.ok(!/curl[^\n]*\/confirm/.test(md), 'carries no ready-to-paste confirm curl');
50
50
  });
@@ -2,7 +2,7 @@
2
2
  // the four health axes from the grader-hardening evaluation §6 (task 1002656):
3
3
  // outage runs, pass-rate drift, cost anomalies, and stale model pins — reading
4
4
  // LIVE data surfaces, emitting exact --regrade remediation lines, and staying
5
- // read-only (describe, never confirm/flip — ADR 0162).
5
+ // read-only (describe, never confirm/flip — ADR 0158 §2; no approval step on anyone's ship — ADR 0162).
6
6
  import assert from 'node:assert/strict';
7
7
  import { test } from 'node:test';
8
8
  import { readFileSync, existsSync } from 'node:fs';
@@ -44,5 +44,5 @@ test('/grader-health exists and covers the four health axes on live data', (t) =
44
44
  assert.match(md, /--probe/, 'offers the optional live calibration probe');
45
45
  assert.match(md, /grade-smoke\.js/, 'names the probe script');
46
46
  assert.match(md, /read-only/i, 'states the read-only contract');
47
- assert.match(md, /ADR 0162/, 'anchors the no-new-gates constraint');
47
+ assert.match(md, /ADR 0162/, 'anchors the no-approval-step-on-a-ship constraint');
48
48
  });
@@ -1,319 +0,0 @@
1
- // Retroactive audit script for task #204.
2
- //
3
- // Loads every task shipped in the last `days` (default 2), fetches full
4
- // task data via the authenticated /tasks/:id API, derives diff stats from
5
- // git (one commit per task, found by the `<VERSION> #<id>:` subject prefix),
6
- // then calls src/bongos/grader.js in-process to get a score.
7
- //
8
- // Output: JSON to stdout (--json) or human-readable table (default).
9
- // Audit-only — does NOT write to task_grades. Pre-grader ships keep clean
10
- // history.
11
-
12
- const { execFileSync } = require('node:child_process');
13
- const grader = require('../../modules/grading/grader');
14
- const { cliClient } = require('./cli-lib');
15
-
16
- // EVERY git call here goes through argv, never a shell string (task 1003223 —
17
- // the class task 1002761 named and task 1002526 fixed in docs-entropy).
18
- //
19
- // WHAT WAS WRONG. Five call sites built shell strings. THREE of them interpolated
20
- // a quoted empty `--format=''`; a POSIX shell strips those quotes and git sees an
21
- // empty format, but the shell Node uses on Windows does not, so git receives
22
- // `--format=''` literally and dies with `fatal: invalid --pretty format: ''`.
23
- // Verified on win32 before fixing: the shell-string form throws, the argv form
24
- // returns " 1 file changed, 139 insertions(+)". The correct argv spelling is
25
- // `--format=` — empty value, no quotes, which no shell can mangle because no
26
- // shell is involved. The other two passed `--format='%H %s'`, which survives a
27
- // POSIX shell and arrives with literal quotes elsewhere; same class, same fix.
28
- //
29
- // WHY IT MATTERED MORE THAN A LOUD CRASH. The originating row asked whether the
30
- // try/catch around these calls "degrades to a silently empty audit — if so it is
31
- // the same dangerous shape after all". It did. Every catch returned null or [],
32
- // so on Windows the audit reported no diff stats and zero downstream impact for
33
- // EVERY ship, and looked like a clean run. `gitFails` below keeps the same return
34
- // values but says so on stderr, so an empty audit can never again be mistaken for
35
- // an uneventful one.
36
- //
37
- // One of the five also built a `grep | head | awk` pipeline and pinned
38
- // `shell: '/bin/bash'` to survive; that filtering is now done in JS, so the script
39
- // needs no shell at all.
40
- function gitLines(args, { cwd } = {}) {
41
- return execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })
42
- .split('\n').map((s) => s.trim()).filter(Boolean);
43
- }
44
-
45
- // One place that reports a git failure instead of swallowing it. Returns the
46
- // caller's fallback so behaviour is unchanged; only the silence is.
47
- function gitFails(what, err, fallback) {
48
- const msg = String((err && (err.stderr || err.message)) || err).trim().split('\n')[0];
49
- console.error(`audit-ships: ${what} failed — ${msg}`);
50
- return fallback;
51
- }
52
-
53
- const DAYS = Number(process.env.AUDIT_DAYS || 2);
54
-
55
- // `git log --all` once per RUN, not once per task.
56
- //
57
- // The shell pipeline this replaced ended in `head -1`, which let git stop as soon
58
- // as one subject matched. The argv rewrite has no such early exit — and
59
- // findCommitForTask is called once per shipped task, up to 500 in a run — so a
60
- // naive port turned one bounded call into 500 full history walks. The commit list
61
- // cannot change mid-run, so it is read once and reused. Caught by the grader on
62
- // the first round of this task, which is a fair catch: replacing a shell pipeline
63
- // means inheriting the work its operators were doing, not just its output.
64
- let commitSubjectsCache = null;
65
- function commitSubjects() {
66
- if (commitSubjectsCache) return commitSubjectsCache;
67
- commitSubjectsCache = gitLines(['log', '--all', '--format=%H %s']);
68
- return commitSubjectsCache;
69
- }
70
-
71
- function findCommitForTask(taskId) {
72
- // Matches both subject conventions: the legacy ' #NNN:' prefix and the de-#-ed
73
- // 'task NNN:' form ship.js writes since task 916 (a bare "#NNN" in commit
74
- // subjects autolinked to dead GitHub issue numbers).
75
- //
76
- // The grep|head|awk pipeline this replaces needed `shell: '/bin/bash'` to run at
77
- // all — a hard dependency on a POSIX shell at a fixed path, for filtering JS can
78
- // do. Doing it here also makes the id match EXACT: the old regex was
79
- // interpolated unescaped, so a task id was pasted straight into a regex.
80
- const re = new RegExp(` (?:#|task )${String(taskId).replace(/[^0-9]/g, '')}:`);
81
- let lines;
82
- try {
83
- lines = commitSubjects();
84
- } catch (e) {
85
- return gitFails(`finding the commit for task ${taskId}`, e, null);
86
- }
87
- const hit = lines.find((l) => re.test(l));
88
- return hit ? (hit.split(' ')[0] || null) : null;
89
- }
90
-
91
- function gitDiffStatsForCommit(sha) {
92
- if (!sha) return null;
93
- let raw;
94
- try {
95
- raw = execFileSync('git', ['show', '--shortstat', '--format=', sha], { encoding: 'utf8' });
96
- } catch (e) {
97
- return gitFails(`reading diff stats for ${String(sha).slice(0, 8)}`, e, null);
98
- }
99
- // Example shortstat line: " 15 files changed, 1171 insertions(+), 65 deletions(-)"
100
- const m = raw.match(/(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?/);
101
- if (!m) return null;
102
- return {
103
- files_changed: Number(m[1] || 0),
104
- lines_added: Number(m[2] || 0),
105
- lines_removed: Number(m[3] || 0),
106
- commit_sha: sha,
107
- };
108
- }
109
-
110
- function filesTouchedByCommit(sha) {
111
- if (!sha) return [];
112
- try {
113
- return gitLines(['show', '--name-only', '--format=', sha]);
114
- } catch (e) {
115
- return gitFails(`listing files touched by ${String(sha).slice(0, 8)}`, e, []);
116
- }
117
- }
118
-
119
- // Downstream-impact heuristic: number of distinct commits on main reachable
120
- // from origin/main that landed AFTER `sha` and touched any of `files`. High
121
- // count = subsequent work has built on top of this change, so a "redo"
122
- // risks conflicts. Excludes `sha` itself.
123
- const NO_DOWNSTREAM = Object.freeze({
124
- commits_after_touching_files: 0, distinct_files_touched: 0, sample_subjects: [],
125
- });
126
-
127
- function downstreamImpact(sha, files) {
128
- if (!sha || files.length === 0) return { ...NO_DOWNSTREAM };
129
- // The hand-rolled POSIX shell-quoting this replaces ("'" + escape + "'") was
130
- // both unnecessary and wrong on Windows, where that quoting style is not the
131
- // shell's. In argv form a path with spaces, quotes or a `$` is just an element.
132
- const range = `${sha}..origin/main`;
133
- let lines;
134
- try {
135
- lines = gitLines(['log', range, '--format=%H %s', '--', ...files]);
136
- } catch (e) {
137
- return gitFails(`reading downstream commits after ${String(sha).slice(0, 8)}`, e, { ...NO_DOWNSTREAM });
138
- }
139
- // Distinct downstream files touched
140
- let distinct = 0;
141
- try {
142
- distinct = new Set(gitLines(['log', range, '--name-only', '--format=', '--', ...files])).size;
143
- } catch (e) {
144
- // Same shape as every other call site: the fallback comes FROM gitFails, so
145
- // the value and the report can never drift apart.
146
- distinct = gitFails(`reading downstream files after ${String(sha).slice(0, 8)}`, e, 0);
147
- }
148
- return {
149
- commits_after_touching_files: lines.length,
150
- distinct_files_touched: distinct,
151
- sample_subjects: lines.slice(0, 3).map((l) => l.split(' ').slice(1).join(' ')),
152
- };
153
- }
154
-
155
- async function fetchShipped(days) {
156
- // Derive the public read host from branding (statusOrigin → publicOrigin), never a
157
- // hardcoded founder host (was https://status.example.com — task 1002234). The
158
- // recent-shipped feed is a public endpoint served on the status dashboard host.
159
- const b = (() => { try { return require('../../src/branding').branding(); } catch (_) { return {}; } })();
160
- const d = b.domains || {};
161
- const statusBase = String(d.statusOrigin || d.publicOrigin || '').replace(/\/+$/, '');
162
- const r = await fetch(
163
- `${statusBase}/api/bongos/public/recent-shipped?days=${days}&limit=500`
164
- );
165
- const data = await r.json();
166
- return data.tasks || [];
167
- }
168
-
169
- async function fetchFullTask(id) {
170
- const api = await cliClient();
171
- const r = await api.tasks.getTasksId({ id });
172
- if (!r.ok) return null;
173
- return r.data.task;
174
- }
175
-
176
- async function fetchLearning(taskId) {
177
- // /api/bongos/learnings?task=<id> — we just need body_md.
178
- const api = await cliClient();
179
- const r = await api.learnings.getLearnings({ query: { task: taskId } });
180
- if (!r.ok) return null;
181
- const rows = r.data.learnings || [];
182
- return rows[0] ? { body_md: rows[0].body_md } : null;
183
- }
184
-
185
- async function main() {
186
- console.error(`Loading shipped tasks (last ${DAYS} days)...`);
187
- const shipped = await fetchShipped(DAYS);
188
- console.error(`Got ${shipped.length} shipped tasks. Fetching full data + diffs...`);
189
-
190
- const results = [];
191
- for (const t of shipped) {
192
- const full = await fetchFullTask(t.id);
193
- if (!full) {
194
- console.error(` #${t.id}: could not fetch full task — skip`);
195
- continue;
196
- }
197
- const sha = findCommitForTask(t.id);
198
- const diff = gitDiffStatsForCommit(sha);
199
- const files = filesTouchedByCommit(sha);
200
- const downstream = downstreamImpact(sha, files);
201
- const verification = {
202
- scanner_clean: true, // got shipped → passed
203
- smoke_clean: true,
204
- diff_stats: diff || undefined,
205
- };
206
- const learningRow =
207
- full.kind === 'learning-capture' ? await fetchLearning(full.id) : null;
208
-
209
- // V3.R18 (#209): the runtime grade entry switched to gradeSubagent;
210
- // this retroactive #204 audit keeps using the v1 deterministic scorer
211
- // because the historical signal we're measuring (process audit, not
212
- // subagent quality) is what's in the v1 rubric. The function was
213
- // renamed from grade() to gradeDeterministic() in the same commit.
214
- const g = grader.gradeDeterministic({
215
- task: full,
216
- verification,
217
- actualMinutes: full.actual_minutes,
218
- valueSummary: full.value_summary,
219
- learningRow,
220
- });
221
-
222
- // Categorize failure mode. Only meaningful when g.passed === false.
223
- // - calibration-noise: only criterion under 5 is est_actual_band
224
- // - quality-concern: multiple criteria under 5
225
- // - structural-risk: downstream impact high (≥10 commits or ≥10 distinct files)
226
- let category = null;
227
- if (!g.passed) {
228
- const lowCriteria = Object.entries(g.rubric_scores)
229
- .filter(([_k, v]) => v < 5)
230
- .map(([k]) => k);
231
- const onlyCalibrationLow = lowCriteria.length === 1 && lowCriteria[0] === 'est_actual_band';
232
- const heavyDownstream =
233
- downstream.commits_after_touching_files >= 10 ||
234
- downstream.distinct_files_touched >= 10;
235
- if (heavyDownstream) category = 'structural-risk';
236
- else if (onlyCalibrationLow) category = 'calibration-noise';
237
- else category = 'quality-concern';
238
- }
239
-
240
- results.push({
241
- id: full.id,
242
- title: full.title,
243
- version_id: full.version_id,
244
- kind: full.kind,
245
- discipline: full.discipline,
246
- est_minutes: full.est_minutes,
247
- actual_minutes: full.actual_minutes,
248
- scoping_confidence: full.scoping_confidence,
249
- value_summary: full.value_summary,
250
- description: full.description,
251
- shipped_at: t.shipped_at,
252
- diff_stats: diff,
253
- files_touched: files,
254
- downstream,
255
- grade: g,
256
- category,
257
- });
258
- }
259
-
260
- if (process.argv.includes('--json')) {
261
- console.log(JSON.stringify(results, null, 2));
262
- return;
263
- }
264
-
265
- // Human-readable summary.
266
- const passed = results.filter((r) => r.grade.passed);
267
- const failed = results.filter((r) => !r.grade.passed);
268
- console.log(`\n=== Audit summary ===`);
269
- console.log(`Total graded: ${results.length}`);
270
- console.log(`Passed: ${passed.length}`);
271
- console.log(`Failed: ${failed.length} (threshold ${grader.RUBRIC.v1.threshold}, v1 deterministic)`);
272
- console.log('');
273
-
274
- if (failed.length === 0) {
275
- console.log('No failures. Nothing to redo.');
276
- return;
277
- }
278
-
279
- console.log('=== FAILURES (sorted by score asc) ===\n');
280
- failed
281
- .sort((a, b) => a.grade.score - b.grade.score)
282
- .forEach((r) => {
283
- console.log(`#${r.id} score ${r.grade.score}/10 [${r.kind}] est=${r.est_minutes}m act=${r.actual_minutes}m`);
284
- console.log(` ${r.title}`);
285
- const scores = r.grade.rubric_scores;
286
- const parts = Object.entries(scores).map(([k, v]) => `${k}=${v}`);
287
- console.log(` ${parts.join(' ')}`);
288
- console.log('');
289
- });
290
-
291
- // Score histogram for the passed set so we see what the distribution looks like.
292
- const buckets = { '5.0-5.9': 0, '6.0-6.9': 0, '7.0-7.9': 0, '8.0-8.9': 0, '9.0-10': 0 };
293
- for (const r of passed) {
294
- const s = r.grade.score;
295
- if (s < 6) buckets['5.0-5.9']++;
296
- else if (s < 7) buckets['6.0-6.9']++;
297
- else if (s < 8) buckets['7.0-7.9']++;
298
- else if (s < 9) buckets['8.0-8.9']++;
299
- else buckets['9.0-10']++;
300
- }
301
- console.log('=== Passed-score histogram ===');
302
- for (const [k, v] of Object.entries(buckets)) console.log(` ${k}: ${v}`);
303
- }
304
-
305
- // Run only as a CLI. Without this guard the file executes main() — and hits the
306
- // API — the instant anything requires it, which is why its git helpers had never
307
- // been tested: there was no way to reach them without starting an audit
308
- // (task 1003223).
309
- if (require.main === module) {
310
- main().catch((err) => {
311
- console.error(err);
312
- process.exit(1);
313
- });
314
- }
315
-
316
- // Exported for tests/audit_ships_git_argv.mjs — the argv-vs-shell-string fix is a
317
- // property of THESE functions, so the test drives them rather than re-typing an
318
- // equivalent git call and hoping it matches.
319
- module.exports = { findCommitForTask, gitDiffStatsForCommit, filesTouchedByCommit, downstreamImpact };