@bongos/core 1.19.1036 → 1.19.1037
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +43 -48
- package/.claude/skills/grade-recover/SKILL.md +1 -1
- package/docs/adr/0302-the-third-identical-failure-stops-the-session-and-asks.md +3 -2
- package/docs/adr/0329-the-rendered-fit-check-is-a-kit-floor-and-an-advisory-ship-check-row.md +2 -2
- package/docs/adr/README.md +1 -1
- package/docs/module-api-changelog.md +2 -0
- package/modules/economy/credits.js +1 -1
- package/modules/grading/grader-config.js +3 -12
- package/modules/grading/grader-no-artifact.js +227 -2
- package/modules/grading/grader-prompt.js +1 -1
- package/modules/grading/grader-rubric.json +1 -86
- package/modules/grading/grader-score.js +1 -169
- package/modules/grading/grader-subagent.js +1 -51
- package/modules/grading/grader.js +26 -211
- package/modules/lifecycle/db-ship.js +3 -3
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +10 -0
- package/scripts/gds/claim.js +2 -1
- package/scripts/gds/grade-smoke.js +3 -3
- package/scripts/gds/render-check.js +1 -1
- package/scripts/gds/ship-check.js +1 -1
- package/scripts/gds/ship-flow.js +1 -1
- package/scripts/gds/ship-grade-steps.js +2 -2
- package/scripts/gds/visual-review-nudge.js +2 -1
- package/src/module-api.js +1 -1
- package/tests/adr_0162_gate_citation.mjs +167 -0
- package/tests/grader.mjs +48 -126
- package/tests/grader_unbacked_score.mjs +1 -1
- package/tests/regrade_actual_minutes.mjs +18 -17
- package/tests/render_check.mjs +1 -1
- package/tests/skill_grade_audit.mjs +3 -3
- package/tests/skill_grader_health.mjs +2 -2
- package/scripts/gds/audit-ships.js +0 -319
- package/tests/audit_ships_git_argv.mjs +0 -173
package/tests/grader.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// tests/grader.mjs
|
|
2
2
|
//
|
|
3
|
-
// Grader tests.
|
|
4
|
-
//
|
|
5
|
-
//
|
|
3
|
+
// Grader tests: the subagent quality grader (the credits gate since V3.R18 #209).
|
|
4
|
+
// The v1 deterministic process-audit grader was retired in task 1002842; the
|
|
5
|
+
// export-surface test below pins that it stays gone.
|
|
6
6
|
//
|
|
7
7
|
// The v2 subagent itself is a real-network call, so the tests stop at the
|
|
8
8
|
// validator + prompt-builder boundary; the network-touching end-to-end check
|
|
@@ -48,127 +48,6 @@ async function testAsync(name, fn) {
|
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
// ========================================================================
|
|
52
|
-
// v1 — deterministic grader (legacy, used only by audit-ships.js)
|
|
53
|
-
// ========================================================================
|
|
54
|
-
|
|
55
|
-
test('v1 scoreEstActualBand: ratio inside [0.5, 2.0] → 10', () => {
|
|
56
|
-
assert.equal(grader.scoreEstActualBand(60, 60), 10);
|
|
57
|
-
assert.equal(grader.scoreEstActualBand(30, 60), 10);
|
|
58
|
-
assert.equal(grader.scoreEstActualBand(120, 60), 10);
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
test('v1 scoreEstActualBand: ratio outside band drops linearly', () => {
|
|
62
|
-
assert.equal(grader.scoreEstActualBand(18, 60), 5);
|
|
63
|
-
assert.equal(grader.scoreEstActualBand(360, 60), 5);
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
test('v1 scoreEstActualBand: extreme ratios → 0', () => {
|
|
67
|
-
assert.equal(grader.scoreEstActualBand(6, 60), 0);
|
|
68
|
-
assert.equal(grader.scoreEstActualBand(600, 60), 0);
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
test('v1 scoreEstActualBand: missing est → 6, missing actual → 5', () => {
|
|
72
|
-
assert.equal(grader.scoreEstActualBand(60, null), 6);
|
|
73
|
-
assert.equal(grader.scoreEstActualBand(null, 60), 5);
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
test('v1 scoreValueSummary: maps length + verb', () => {
|
|
77
|
-
assert.equal(grader.scoreValueSummary(''), 0);
|
|
78
|
-
assert.equal(grader.scoreValueSummary(null), 0);
|
|
79
|
-
assert.equal(grader.scoreValueSummary('short note'), 5);
|
|
80
|
-
assert.equal(grader.scoreValueSummary('a moderately worded summary that lands above forty characters'), 8);
|
|
81
|
-
const s = 'wire the auto-grader into the completed→confirmed transition so every task gets a quality mark before its credits are paid out';
|
|
82
|
-
assert.ok(s.length > 120);
|
|
83
|
-
assert.equal(grader.scoreValueSummary(s), 10);
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
test('v1 scoreDiffHygiene: phantom ship → 0, normal → 10, too-broad → 5', () => {
|
|
87
|
-
assert.equal(grader.scoreDiffHygiene({ files_changed: 0, lines_added: 0, lines_removed: 0 }), 0);
|
|
88
|
-
assert.equal(grader.scoreDiffHygiene({ files_changed: 5, lines_added: 200, lines_removed: 30 }), 10);
|
|
89
|
-
assert.equal(grader.scoreDiffHygiene({ files_changed: 80, lines_added: 1000, lines_removed: 200 }), 5);
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
test('v1 scoreScopingSignal: maps each level', () => {
|
|
93
|
-
assert.equal(grader.scoreScopingSignal(0), 2);
|
|
94
|
-
assert.equal(grader.scoreScopingSignal(1), 4);
|
|
95
|
-
assert.equal(grader.scoreScopingSignal(2), 6);
|
|
96
|
-
assert.equal(grader.scoreScopingSignal(3), 7);
|
|
97
|
-
assert.equal(grader.scoreScopingSignal(4), 8);
|
|
98
|
-
assert.equal(grader.scoreScopingSignal(5), 10);
|
|
99
|
-
assert.equal(grader.scoreScopingSignal(null), 6);
|
|
100
|
-
});
|
|
101
|
-
|
|
102
|
-
test('v1 scoreLocNetNegative', () => {
|
|
103
|
-
assert.equal(grader.scoreLocNetNegative({ lines_added: 20, lines_removed: 100 }), 10);
|
|
104
|
-
assert.equal(grader.scoreLocNetNegative({ lines_added: 100, lines_removed: 20 }), 0);
|
|
105
|
-
});
|
|
106
|
-
|
|
107
|
-
test('v1 scoreLearningRow', () => {
|
|
108
|
-
assert.equal(grader.scoreLearningRow({ body_md: 'x'.repeat(200) }), 10);
|
|
109
|
-
assert.equal(grader.scoreLearningRow(null), 0);
|
|
110
|
-
});
|
|
111
|
-
|
|
112
|
-
test('v1 scoreArtRubricAvg clamps + nulls', () => {
|
|
113
|
-
assert.equal(grader.scoreArtRubricAvg(8.7), 8.7);
|
|
114
|
-
assert.equal(grader.scoreArtRubricAvg(15), 10);
|
|
115
|
-
assert.equal(grader.scoreArtRubricAvg(-1), 0);
|
|
116
|
-
assert.equal(grader.scoreArtRubricAvg(NaN), null);
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
test('v1 blendWeightsForKind: feature renorms to 1.0', () => {
|
|
120
|
-
const w = grader.blendWeightsForKind('feature');
|
|
121
|
-
const total = Object.values(w).reduce((a, b) => a + b, 0);
|
|
122
|
-
assert.ok(Math.abs(total - 1.0) < 1e-6, `total=${total}, expected 1.0`);
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
test('v1 blendWeightsForKind: cleanup replaces diff_hygiene', () => {
|
|
126
|
-
const w = grader.blendWeightsForKind('cleanup');
|
|
127
|
-
assert.ok('loc_net_negative' in w);
|
|
128
|
-
assert.ok(!('diff_hygiene' in w));
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
test('v1 gradeDeterministic: well-shipped feature passes', () => {
|
|
132
|
-
const result = grader.gradeDeterministic({
|
|
133
|
-
task: { id: 99, kind: 'feature', est_minutes: 240, actual_minutes: 200, scoping_confidence: 4 },
|
|
134
|
-
verification: {
|
|
135
|
-
scanner_clean: true, smoke_clean: true,
|
|
136
|
-
diff_stats: { files_changed: 10, lines_added: 400, lines_removed: 50 },
|
|
137
|
-
},
|
|
138
|
-
actualMinutes: 200,
|
|
139
|
-
valueSummary: 'wire the auto-grader into ship.js so every task gets a quality mark before credits are paid; gates the completed→confirmed transition',
|
|
140
|
-
});
|
|
141
|
-
assert.ok(result.score >= 9.0, `score=${result.score}`);
|
|
142
|
-
assert.equal(result.passed, true);
|
|
143
|
-
assert.equal(result.grader_kind, 'auto');
|
|
144
|
-
assert.equal(result.grader_version, grader.RUBRIC.v1.version);
|
|
145
|
-
assert.equal(result.threshold, grader.RUBRIC.v1.threshold);
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
test('v1 gradeDeterministic: phantom ship scores low', () => {
|
|
149
|
-
const result = grader.gradeDeterministic({
|
|
150
|
-
task: { id: 100, kind: 'feature', est_minutes: 240, actual_minutes: 30, scoping_confidence: 2 },
|
|
151
|
-
verification: {
|
|
152
|
-
scanner_clean: true, smoke_clean: true,
|
|
153
|
-
diff_stats: { files_changed: 0, lines_added: 0, lines_removed: 0 },
|
|
154
|
-
},
|
|
155
|
-
actualMinutes: 30,
|
|
156
|
-
valueSummary: '',
|
|
157
|
-
});
|
|
158
|
-
assert.ok(result.score < 5.0, `score=${result.score}`);
|
|
159
|
-
assert.equal(result.passed, false);
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
test('v1 gradeDeterministic: notes_md says deprecated', () => {
|
|
163
|
-
const result = grader.gradeDeterministic({
|
|
164
|
-
task: { id: 106, kind: 'feature', est_minutes: 60, actual_minutes: 60, scoping_confidence: 4 },
|
|
165
|
-
verification: { scanner_clean: true, smoke_clean: true, diff_stats: { files_changed: 3, lines_added: 50, lines_removed: 5 } },
|
|
166
|
-
actualMinutes: 60,
|
|
167
|
-
valueSummary: 'a reasonable shipped change',
|
|
168
|
-
});
|
|
169
|
-
assert.match(result.notes_md, /v1.*deprecated/i);
|
|
170
|
-
});
|
|
171
|
-
|
|
172
51
|
// ========================================================================
|
|
173
52
|
// v2 — subagent grader (the credits gate as of V3.R18 #209)
|
|
174
53
|
// ========================================================================
|
|
@@ -181,17 +60,29 @@ test('v4 exports the expected surface (current panel-based gate)', () => {
|
|
|
181
60
|
'buildPrompt', 'selectGraderModel', 'validateAndScore', 'aggregateShots',
|
|
182
61
|
'runSubagent', 'runOneShot', 'gradeSubagentV3MultiShot',
|
|
183
62
|
// constants
|
|
184
|
-
'THRESHOLD', 'VERSION', 'DEFAULT_GRADER_MODEL', '
|
|
63
|
+
'THRESHOLD', 'VERSION', 'DEFAULT_GRADER_MODEL', 'ASSUMED_BUILDER_MODEL',
|
|
185
64
|
// utilities
|
|
186
65
|
'extractJson', 'extractAssistantResult', 'extractCostUsd', 'RUBRIC',
|
|
187
66
|
];
|
|
188
67
|
for (const name of required) {
|
|
189
68
|
assert.ok(name in grader, `missing export: ${name}`);
|
|
190
69
|
}
|
|
70
|
+
// task 1002842: the v1 generation is retired, not merely unused.
|
|
71
|
+
for (const gone of ['gradeDeterministic', 'scoreEstActualBand', 'scoreValueSummary', 'scoreDiffHygiene',
|
|
72
|
+
'scoreScannerSmoke', 'scoreScopingSignal', 'scoreLocNetNegative', 'scoreLearningRow', 'scoreArtRubricAvg',
|
|
73
|
+
'blendWeightsForKind']) {
|
|
74
|
+
assert.equal(gone in grader, false, `retired v1 export came back: ${gone}`);
|
|
75
|
+
}
|
|
76
|
+
assert.equal('v1' in grader.RUBRIC, false, 'the v1 rubric block is retired');
|
|
77
|
+
assert.equal('v3' in grader.RUBRIC, false, 'the v3 rubric block is retired');
|
|
78
|
+
// The moved plumbing is one implementation, reachable from both homes.
|
|
79
|
+
const noArtifact = require('../modules/grading/grader-no-artifact.js');
|
|
80
|
+
assert.equal(grader.aggregateShots, noArtifact.aggregateShots);
|
|
81
|
+
assert.equal(grader.runOneShot, noArtifact.runOneShot);
|
|
191
82
|
assert.ok(typeof grader.THRESHOLD === 'number');
|
|
192
83
|
assert.ok(grader.THRESHOLD >= 5.0 && grader.THRESHOLD <= 10.0);
|
|
193
84
|
assert.ok(grader.VERSION.startsWith('v4'), `expected v4.*, got ${grader.VERSION}`);
|
|
194
|
-
assert.equal(
|
|
85
|
+
assert.equal('DEFAULT_SHOTS' in grader, false, 'the v3 3-shot default is retired (task 1002842)');
|
|
195
86
|
assert.equal(grader.DEFAULT_GRADER_MODEL, 'claude-sonnet-5'); // task 1002659: Claude 5 family pin
|
|
196
87
|
assert.equal(grader.ASSUMED_BUILDER_MODEL, 'claude-fable-5');
|
|
197
88
|
assert.equal(grader.FALLTHROUGH_CONFIDENCE, 0.3);
|
|
@@ -2316,6 +2207,37 @@ test('isTrivialDiff: empty changed-files or zero-line diff → false', () => {
|
|
|
2316
2207
|
);
|
|
2317
2208
|
});
|
|
2318
2209
|
|
|
2210
|
+
// ---------- task 1002842: the kept trivial-diff fast path still works ----------
|
|
2211
|
+
//
|
|
2212
|
+
// The v3 generation is retired, but its fast path is kept (moving it onto a v4
|
|
2213
|
+
// panel needs a new reviewer type — an owner decision). These pin that what was
|
|
2214
|
+
// kept works: ONE generalist shot by default, the generalist prompt, and the
|
|
2215
|
+
// moved aggregateShots collapse — driven through the runShot seam, no model spawned.
|
|
2216
|
+
|
|
2217
|
+
await testAsync('fast path: one generalist shot by default, aggregated into a gradeable result', async () => {
|
|
2218
|
+
const calls = [];
|
|
2219
|
+
const r = await grader.gradeSubagentV3MultiShot({
|
|
2220
|
+
task: { id: 1002842, title: 't', description: 'd', kind: 'cleanup' },
|
|
2221
|
+
valueSummary: 'v', diff: '+ docs line', changedFiles: ['docs/x.md'], diffStats: { files_changed: 1, lines_added: 1, lines_removed: 0 },
|
|
2222
|
+
builderModel: 'claude-fable-5', runOpts: {},
|
|
2223
|
+
runShot: async (args) => { calls.push(args); return mkPassShot(8, 8, 0.9, args.shotIndex); },
|
|
2224
|
+
});
|
|
2225
|
+
assert.equal(calls.length, 1, 'the fast path runs exactly one shot unless told otherwise');
|
|
2226
|
+
assert.match(calls[0].prompt, /default-fail|DEFAULT/i, 'it is the generalist default-fail prompt');
|
|
2227
|
+
assert.equal(r.passed, true);
|
|
2228
|
+
assert.equal(r.rubric_scores.functional_fidelity, 8);
|
|
2229
|
+
assert.equal(r.rubric_scores.code_quality, 8);
|
|
2230
|
+
assert.equal(r.signals.confidence, 0.9, 'the server gate reads a real confidence');
|
|
2231
|
+
});
|
|
2232
|
+
|
|
2233
|
+
await testAsync('fast path: a failing shot still fails the grade (not loosened by the move)', async () => {
|
|
2234
|
+
const r = await grader.gradeSubagentV3MultiShot({
|
|
2235
|
+
task: { id: 1 }, valueSummary: 'v', diff: '+ x', changedFiles: ['docs/x.md'], diffStats: {},
|
|
2236
|
+
runOpts: {}, runShot: async (args) => mkFailShot(4, 8, 0.9, args.shotIndex),
|
|
2237
|
+
});
|
|
2238
|
+
assert.equal(r.passed, false);
|
|
2239
|
+
});
|
|
2240
|
+
|
|
2319
2241
|
// ---------- Summary ----------
|
|
2320
2242
|
|
|
2321
2243
|
console.log('');
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// server's ff gate failed the ship — a PASS verdict inside a FAIL grade;
|
|
15
15
|
// 3. the card had no branch for it, so it printed the ordinary FAIL block.
|
|
16
16
|
// The constraint the task set: close it WITHOUT letting an unbacked low score
|
|
17
|
-
// pass and WITHOUT adding a gate (
|
|
17
|
+
// pass and WITHOUT adding a gate (the task's own scope: it corrects what the grade SAYS). Every case below that has a low
|
|
18
18
|
// score also asserts the server still refuses it.
|
|
19
19
|
//
|
|
20
20
|
// Run: node tests/grader_unbacked_score.mjs
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
import { strict as assert } from 'node:assert';
|
|
33
33
|
import { test } from 'node:test';
|
|
34
34
|
import { createRequire } from 'node:module';
|
|
35
|
-
import { readFileSync } from 'node:fs';
|
|
35
|
+
import { readFileSync, readdirSync } from 'node:fs';
|
|
36
36
|
|
|
37
37
|
const require = createRequire(import.meta.url);
|
|
38
38
|
const { buildPrompt } = require('../modules/grading/grader-prompt.js');
|
|
@@ -78,23 +78,24 @@ test('a task with no actual_minutes at all builds the same prompt', () => {
|
|
|
78
78
|
assert.equal(withFigure, without, 'actual_minutes changes nothing about what the panel reads');
|
|
79
79
|
});
|
|
80
80
|
|
|
81
|
-
test('
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
//
|
|
86
|
-
//
|
|
81
|
+
test('no grader entry reads actual_minutes — the one scorer that did is retired', () => {
|
|
82
|
+
// scoreEstActualBand was the ONE scorer that read the figure. It lived in the v1
|
|
83
|
+
// gradeDeterministic, whose only non-test caller was scripts/gds/audit-ships.js;
|
|
84
|
+
// task 1002842 retired all three. So the stale-figure risk this file guards is now
|
|
85
|
+
// closed at the source, and this pins it: nothing in the grading module may
|
|
86
|
+
// accept or read the figure, and the retired entry may not come back.
|
|
87
87
|
const grader = require('../modules/grading/grader.js');
|
|
88
|
-
assert.equal(
|
|
89
|
-
assert.equal(grader.
|
|
88
|
+
assert.equal(grader.gradeDeterministic, undefined, 'the retired v1 entry stays gone');
|
|
89
|
+
assert.equal(grader.scoreEstActualBand, undefined, 'and so does its minutes scorer');
|
|
90
|
+
assert.equal(typeof grader.gradeSubagent, 'function', 'the live entry is still here');
|
|
90
91
|
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
assert.
|
|
92
|
+
// Every grading source, not just grader.js: a new reader anywhere in the module
|
|
93
|
+
// is the same risk as wiring the figure into the prompt.
|
|
94
|
+
const dir = new URL('../modules/grading/', import.meta.url);
|
|
95
|
+
const files = readdirSync(dir, { recursive: true }).filter((f) => String(f).endsWith('.js'));
|
|
96
|
+
assert.ok(files.length > 5, 'the grading module was found');
|
|
97
|
+
for (const f of files) {
|
|
98
|
+
const src = readFileSync(new URL(String(f), dir), 'utf8');
|
|
99
|
+
assert.doesNotMatch(src, /actualMinutes|actual_minutes/, `${f} must not read actual_minutes`);
|
|
99
100
|
}
|
|
100
101
|
});
|
package/tests/render_check.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// tests/render_check.mjs — the rendered fit check (task 1004203, ADR 0329): which pages a changed path
|
|
2
2
|
// reaches, which report rows it keeps, and that /ship-check shows it as a signal that never flips its exit
|
|
3
|
-
// (ADR
|
|
3
|
+
// (ADR 0329, decision 3). DB-free and browser-free; the in-page measurement is pinned in tests/ui_design_kit_fit.mjs.
|
|
4
4
|
import assert from 'node:assert/strict';
|
|
5
5
|
import { test } from 'node:test';
|
|
6
6
|
import fs from 'node:fs';
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// accountability sweep from the grader-hardening evaluation §6 (task 1002657):
|
|
3
3
|
// the override ledger, dropped blocker/major findings on passing grades (with
|
|
4
4
|
// the advisory-Narc attribution filter), the false-pass spot-check, and the
|
|
5
|
-
// outage roll-up handoff — describing and queueing ONLY (ADR
|
|
6
|
-
// never confirms a task or flips status).
|
|
5
|
+
// outage roll-up handoff — describing and queueing ONLY (ADR 0158 §2: it
|
|
6
|
+
// never confirms a task or flips status; ADR 0162: it adds no approval step to a ship).
|
|
7
7
|
import assert from 'node:assert/strict';
|
|
8
8
|
import { test } from 'node:test';
|
|
9
9
|
import { readFileSync, existsSync } from 'node:fs';
|
|
@@ -45,6 +45,6 @@ test('/grade-audit exists, covers the sweep legs, and stays describe-and-queue o
|
|
|
45
45
|
|
|
46
46
|
// The hard constraint.
|
|
47
47
|
assert.match(md, /never confirm/i, 'states it never confirms');
|
|
48
|
-
assert.match(md, /ADR 0162/, 'anchors the no-
|
|
48
|
+
assert.match(md, /ADR 0162/, 'anchors the no-approval-step-on-a-ship constraint');
|
|
49
49
|
assert.ok(!/curl[^\n]*\/confirm/.test(md), 'carries no ready-to-paste confirm curl');
|
|
50
50
|
});
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the four health axes from the grader-hardening evaluation §6 (task 1002656):
|
|
3
3
|
// outage runs, pass-rate drift, cost anomalies, and stale model pins — reading
|
|
4
4
|
// LIVE data surfaces, emitting exact --regrade remediation lines, and staying
|
|
5
|
-
// read-only (describe, never confirm/flip — ADR 0162).
|
|
5
|
+
// read-only (describe, never confirm/flip — ADR 0158 §2; no approval step on anyone's ship — ADR 0162).
|
|
6
6
|
import assert from 'node:assert/strict';
|
|
7
7
|
import { test } from 'node:test';
|
|
8
8
|
import { readFileSync, existsSync } from 'node:fs';
|
|
@@ -44,5 +44,5 @@ test('/grader-health exists and covers the four health axes on live data', (t) =
|
|
|
44
44
|
assert.match(md, /--probe/, 'offers the optional live calibration probe');
|
|
45
45
|
assert.match(md, /grade-smoke\.js/, 'names the probe script');
|
|
46
46
|
assert.match(md, /read-only/i, 'states the read-only contract');
|
|
47
|
-
assert.match(md, /ADR 0162/, 'anchors the no-
|
|
47
|
+
assert.match(md, /ADR 0162/, 'anchors the no-approval-step-on-a-ship constraint');
|
|
48
48
|
});
|
|
@@ -1,319 +0,0 @@
|
|
|
1
|
-
// Retroactive audit script for task #204.
|
|
2
|
-
//
|
|
3
|
-
// Loads every task shipped in the last `days` (default 2), fetches full
|
|
4
|
-
// task data via the authenticated /tasks/:id API, derives diff stats from
|
|
5
|
-
// git (one commit per task, found by the `<VERSION> #<id>:` subject prefix),
|
|
6
|
-
// then calls src/bongos/grader.js in-process to get a score.
|
|
7
|
-
//
|
|
8
|
-
// Output: JSON to stdout (--json) or human-readable table (default).
|
|
9
|
-
// Audit-only — does NOT write to task_grades. Pre-grader ships keep clean
|
|
10
|
-
// history.
|
|
11
|
-
|
|
12
|
-
const { execFileSync } = require('node:child_process');
|
|
13
|
-
const grader = require('../../modules/grading/grader');
|
|
14
|
-
const { cliClient } = require('./cli-lib');
|
|
15
|
-
|
|
16
|
-
// EVERY git call here goes through argv, never a shell string (task 1003223 —
|
|
17
|
-
// the class task 1002761 named and task 1002526 fixed in docs-entropy).
|
|
18
|
-
//
|
|
19
|
-
// WHAT WAS WRONG. Five call sites built shell strings. THREE of them interpolated
|
|
20
|
-
// a quoted empty `--format=''`; a POSIX shell strips those quotes and git sees an
|
|
21
|
-
// empty format, but the shell Node uses on Windows does not, so git receives
|
|
22
|
-
// `--format=''` literally and dies with `fatal: invalid --pretty format: ''`.
|
|
23
|
-
// Verified on win32 before fixing: the shell-string form throws, the argv form
|
|
24
|
-
// returns " 1 file changed, 139 insertions(+)". The correct argv spelling is
|
|
25
|
-
// `--format=` — empty value, no quotes, which no shell can mangle because no
|
|
26
|
-
// shell is involved. The other two passed `--format='%H %s'`, which survives a
|
|
27
|
-
// POSIX shell and arrives with literal quotes elsewhere; same class, same fix.
|
|
28
|
-
//
|
|
29
|
-
// WHY IT MATTERED MORE THAN A LOUD CRASH. The originating row asked whether the
|
|
30
|
-
// try/catch around these calls "degrades to a silently empty audit — if so it is
|
|
31
|
-
// the same dangerous shape after all". It did. Every catch returned null or [],
|
|
32
|
-
// so on Windows the audit reported no diff stats and zero downstream impact for
|
|
33
|
-
// EVERY ship, and looked like a clean run. `gitFails` below keeps the same return
|
|
34
|
-
// values but says so on stderr, so an empty audit can never again be mistaken for
|
|
35
|
-
// an uneventful one.
|
|
36
|
-
//
|
|
37
|
-
// One of the five also built a `grep | head | awk` pipeline and pinned
|
|
38
|
-
// `shell: '/bin/bash'` to survive; that filtering is now done in JS, so the script
|
|
39
|
-
// needs no shell at all.
|
|
40
|
-
function gitLines(args, { cwd } = {}) {
|
|
41
|
-
return execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })
|
|
42
|
-
.split('\n').map((s) => s.trim()).filter(Boolean);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
// One place that reports a git failure instead of swallowing it. Returns the
|
|
46
|
-
// caller's fallback so behaviour is unchanged; only the silence is.
|
|
47
|
-
function gitFails(what, err, fallback) {
|
|
48
|
-
const msg = String((err && (err.stderr || err.message)) || err).trim().split('\n')[0];
|
|
49
|
-
console.error(`audit-ships: ${what} failed — ${msg}`);
|
|
50
|
-
return fallback;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
const DAYS = Number(process.env.AUDIT_DAYS || 2);
|
|
54
|
-
|
|
55
|
-
// `git log --all` once per RUN, not once per task.
|
|
56
|
-
//
|
|
57
|
-
// The shell pipeline this replaced ended in `head -1`, which let git stop as soon
|
|
58
|
-
// as one subject matched. The argv rewrite has no such early exit — and
|
|
59
|
-
// findCommitForTask is called once per shipped task, up to 500 in a run — so a
|
|
60
|
-
// naive port turned one bounded call into 500 full history walks. The commit list
|
|
61
|
-
// cannot change mid-run, so it is read once and reused. Caught by the grader on
|
|
62
|
-
// the first round of this task, which is a fair catch: replacing a shell pipeline
|
|
63
|
-
// means inheriting the work its operators were doing, not just its output.
|
|
64
|
-
let commitSubjectsCache = null;
|
|
65
|
-
function commitSubjects() {
|
|
66
|
-
if (commitSubjectsCache) return commitSubjectsCache;
|
|
67
|
-
commitSubjectsCache = gitLines(['log', '--all', '--format=%H %s']);
|
|
68
|
-
return commitSubjectsCache;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
function findCommitForTask(taskId) {
|
|
72
|
-
// Matches both subject conventions: the legacy ' #NNN:' prefix and the de-#-ed
|
|
73
|
-
// 'task NNN:' form ship.js writes since task 916 (a bare "#NNN" in commit
|
|
74
|
-
// subjects autolinked to dead GitHub issue numbers).
|
|
75
|
-
//
|
|
76
|
-
// The grep|head|awk pipeline this replaces needed `shell: '/bin/bash'` to run at
|
|
77
|
-
// all — a hard dependency on a POSIX shell at a fixed path, for filtering JS can
|
|
78
|
-
// do. Doing it here also makes the id match EXACT: the old regex was
|
|
79
|
-
// interpolated unescaped, so a task id was pasted straight into a regex.
|
|
80
|
-
const re = new RegExp(` (?:#|task )${String(taskId).replace(/[^0-9]/g, '')}:`);
|
|
81
|
-
let lines;
|
|
82
|
-
try {
|
|
83
|
-
lines = commitSubjects();
|
|
84
|
-
} catch (e) {
|
|
85
|
-
return gitFails(`finding the commit for task ${taskId}`, e, null);
|
|
86
|
-
}
|
|
87
|
-
const hit = lines.find((l) => re.test(l));
|
|
88
|
-
return hit ? (hit.split(' ')[0] || null) : null;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function gitDiffStatsForCommit(sha) {
|
|
92
|
-
if (!sha) return null;
|
|
93
|
-
let raw;
|
|
94
|
-
try {
|
|
95
|
-
raw = execFileSync('git', ['show', '--shortstat', '--format=', sha], { encoding: 'utf8' });
|
|
96
|
-
} catch (e) {
|
|
97
|
-
return gitFails(`reading diff stats for ${String(sha).slice(0, 8)}`, e, null);
|
|
98
|
-
}
|
|
99
|
-
// Example shortstat line: " 15 files changed, 1171 insertions(+), 65 deletions(-)"
|
|
100
|
-
const m = raw.match(/(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?/);
|
|
101
|
-
if (!m) return null;
|
|
102
|
-
return {
|
|
103
|
-
files_changed: Number(m[1] || 0),
|
|
104
|
-
lines_added: Number(m[2] || 0),
|
|
105
|
-
lines_removed: Number(m[3] || 0),
|
|
106
|
-
commit_sha: sha,
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
function filesTouchedByCommit(sha) {
|
|
111
|
-
if (!sha) return [];
|
|
112
|
-
try {
|
|
113
|
-
return gitLines(['show', '--name-only', '--format=', sha]);
|
|
114
|
-
} catch (e) {
|
|
115
|
-
return gitFails(`listing files touched by ${String(sha).slice(0, 8)}`, e, []);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
// Downstream-impact heuristic: number of distinct commits on main reachable
|
|
120
|
-
// from origin/main that landed AFTER `sha` and touched any of `files`. High
|
|
121
|
-
// count = subsequent work has built on top of this change, so a "redo"
|
|
122
|
-
// risks conflicts. Excludes `sha` itself.
|
|
123
|
-
const NO_DOWNSTREAM = Object.freeze({
|
|
124
|
-
commits_after_touching_files: 0, distinct_files_touched: 0, sample_subjects: [],
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
function downstreamImpact(sha, files) {
|
|
128
|
-
if (!sha || files.length === 0) return { ...NO_DOWNSTREAM };
|
|
129
|
-
// The hand-rolled POSIX shell-quoting this replaces ("'" + escape + "'") was
|
|
130
|
-
// both unnecessary and wrong on Windows, where that quoting style is not the
|
|
131
|
-
// shell's. In argv form a path with spaces, quotes or a `$` is just an element.
|
|
132
|
-
const range = `${sha}..origin/main`;
|
|
133
|
-
let lines;
|
|
134
|
-
try {
|
|
135
|
-
lines = gitLines(['log', range, '--format=%H %s', '--', ...files]);
|
|
136
|
-
} catch (e) {
|
|
137
|
-
return gitFails(`reading downstream commits after ${String(sha).slice(0, 8)}`, e, { ...NO_DOWNSTREAM });
|
|
138
|
-
}
|
|
139
|
-
// Distinct downstream files touched
|
|
140
|
-
let distinct = 0;
|
|
141
|
-
try {
|
|
142
|
-
distinct = new Set(gitLines(['log', range, '--name-only', '--format=', '--', ...files])).size;
|
|
143
|
-
} catch (e) {
|
|
144
|
-
// Same shape as every other call site: the fallback comes FROM gitFails, so
|
|
145
|
-
// the value and the report can never drift apart.
|
|
146
|
-
distinct = gitFails(`reading downstream files after ${String(sha).slice(0, 8)}`, e, 0);
|
|
147
|
-
}
|
|
148
|
-
return {
|
|
149
|
-
commits_after_touching_files: lines.length,
|
|
150
|
-
distinct_files_touched: distinct,
|
|
151
|
-
sample_subjects: lines.slice(0, 3).map((l) => l.split(' ').slice(1).join(' ')),
|
|
152
|
-
};
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
async function fetchShipped(days) {
|
|
156
|
-
// Derive the public read host from branding (statusOrigin → publicOrigin), never a
|
|
157
|
-
// hardcoded founder host (was https://status.example.com — task 1002234). The
|
|
158
|
-
// recent-shipped feed is a public endpoint served on the status dashboard host.
|
|
159
|
-
const b = (() => { try { return require('../../src/branding').branding(); } catch (_) { return {}; } })();
|
|
160
|
-
const d = b.domains || {};
|
|
161
|
-
const statusBase = String(d.statusOrigin || d.publicOrigin || '').replace(/\/+$/, '');
|
|
162
|
-
const r = await fetch(
|
|
163
|
-
`${statusBase}/api/bongos/public/recent-shipped?days=${days}&limit=500`
|
|
164
|
-
);
|
|
165
|
-
const data = await r.json();
|
|
166
|
-
return data.tasks || [];
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
async function fetchFullTask(id) {
|
|
170
|
-
const api = await cliClient();
|
|
171
|
-
const r = await api.tasks.getTasksId({ id });
|
|
172
|
-
if (!r.ok) return null;
|
|
173
|
-
return r.data.task;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
async function fetchLearning(taskId) {
|
|
177
|
-
// /api/bongos/learnings?task=<id> — we just need body_md.
|
|
178
|
-
const api = await cliClient();
|
|
179
|
-
const r = await api.learnings.getLearnings({ query: { task: taskId } });
|
|
180
|
-
if (!r.ok) return null;
|
|
181
|
-
const rows = r.data.learnings || [];
|
|
182
|
-
return rows[0] ? { body_md: rows[0].body_md } : null;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
async function main() {
|
|
186
|
-
console.error(`Loading shipped tasks (last ${DAYS} days)...`);
|
|
187
|
-
const shipped = await fetchShipped(DAYS);
|
|
188
|
-
console.error(`Got ${shipped.length} shipped tasks. Fetching full data + diffs...`);
|
|
189
|
-
|
|
190
|
-
const results = [];
|
|
191
|
-
for (const t of shipped) {
|
|
192
|
-
const full = await fetchFullTask(t.id);
|
|
193
|
-
if (!full) {
|
|
194
|
-
console.error(` #${t.id}: could not fetch full task — skip`);
|
|
195
|
-
continue;
|
|
196
|
-
}
|
|
197
|
-
const sha = findCommitForTask(t.id);
|
|
198
|
-
const diff = gitDiffStatsForCommit(sha);
|
|
199
|
-
const files = filesTouchedByCommit(sha);
|
|
200
|
-
const downstream = downstreamImpact(sha, files);
|
|
201
|
-
const verification = {
|
|
202
|
-
scanner_clean: true, // got shipped → passed
|
|
203
|
-
smoke_clean: true,
|
|
204
|
-
diff_stats: diff || undefined,
|
|
205
|
-
};
|
|
206
|
-
const learningRow =
|
|
207
|
-
full.kind === 'learning-capture' ? await fetchLearning(full.id) : null;
|
|
208
|
-
|
|
209
|
-
// V3.R18 (#209): the runtime grade entry switched to gradeSubagent;
|
|
210
|
-
// this retroactive #204 audit keeps using the v1 deterministic scorer
|
|
211
|
-
// because the historical signal we're measuring (process audit, not
|
|
212
|
-
// subagent quality) is what's in the v1 rubric. The function was
|
|
213
|
-
// renamed from grade() to gradeDeterministic() in the same commit.
|
|
214
|
-
const g = grader.gradeDeterministic({
|
|
215
|
-
task: full,
|
|
216
|
-
verification,
|
|
217
|
-
actualMinutes: full.actual_minutes,
|
|
218
|
-
valueSummary: full.value_summary,
|
|
219
|
-
learningRow,
|
|
220
|
-
});
|
|
221
|
-
|
|
222
|
-
// Categorize failure mode. Only meaningful when g.passed === false.
|
|
223
|
-
// - calibration-noise: only criterion under 5 is est_actual_band
|
|
224
|
-
// - quality-concern: multiple criteria under 5
|
|
225
|
-
// - structural-risk: downstream impact high (≥10 commits or ≥10 distinct files)
|
|
226
|
-
let category = null;
|
|
227
|
-
if (!g.passed) {
|
|
228
|
-
const lowCriteria = Object.entries(g.rubric_scores)
|
|
229
|
-
.filter(([_k, v]) => v < 5)
|
|
230
|
-
.map(([k]) => k);
|
|
231
|
-
const onlyCalibrationLow = lowCriteria.length === 1 && lowCriteria[0] === 'est_actual_band';
|
|
232
|
-
const heavyDownstream =
|
|
233
|
-
downstream.commits_after_touching_files >= 10 ||
|
|
234
|
-
downstream.distinct_files_touched >= 10;
|
|
235
|
-
if (heavyDownstream) category = 'structural-risk';
|
|
236
|
-
else if (onlyCalibrationLow) category = 'calibration-noise';
|
|
237
|
-
else category = 'quality-concern';
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
results.push({
|
|
241
|
-
id: full.id,
|
|
242
|
-
title: full.title,
|
|
243
|
-
version_id: full.version_id,
|
|
244
|
-
kind: full.kind,
|
|
245
|
-
discipline: full.discipline,
|
|
246
|
-
est_minutes: full.est_minutes,
|
|
247
|
-
actual_minutes: full.actual_minutes,
|
|
248
|
-
scoping_confidence: full.scoping_confidence,
|
|
249
|
-
value_summary: full.value_summary,
|
|
250
|
-
description: full.description,
|
|
251
|
-
shipped_at: t.shipped_at,
|
|
252
|
-
diff_stats: diff,
|
|
253
|
-
files_touched: files,
|
|
254
|
-
downstream,
|
|
255
|
-
grade: g,
|
|
256
|
-
category,
|
|
257
|
-
});
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
if (process.argv.includes('--json')) {
|
|
261
|
-
console.log(JSON.stringify(results, null, 2));
|
|
262
|
-
return;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
// Human-readable summary.
|
|
266
|
-
const passed = results.filter((r) => r.grade.passed);
|
|
267
|
-
const failed = results.filter((r) => !r.grade.passed);
|
|
268
|
-
console.log(`\n=== Audit summary ===`);
|
|
269
|
-
console.log(`Total graded: ${results.length}`);
|
|
270
|
-
console.log(`Passed: ${passed.length}`);
|
|
271
|
-
console.log(`Failed: ${failed.length} (threshold ${grader.RUBRIC.v1.threshold}, v1 deterministic)`);
|
|
272
|
-
console.log('');
|
|
273
|
-
|
|
274
|
-
if (failed.length === 0) {
|
|
275
|
-
console.log('No failures. Nothing to redo.');
|
|
276
|
-
return;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
console.log('=== FAILURES (sorted by score asc) ===\n');
|
|
280
|
-
failed
|
|
281
|
-
.sort((a, b) => a.grade.score - b.grade.score)
|
|
282
|
-
.forEach((r) => {
|
|
283
|
-
console.log(`#${r.id} score ${r.grade.score}/10 [${r.kind}] est=${r.est_minutes}m act=${r.actual_minutes}m`);
|
|
284
|
-
console.log(` ${r.title}`);
|
|
285
|
-
const scores = r.grade.rubric_scores;
|
|
286
|
-
const parts = Object.entries(scores).map(([k, v]) => `${k}=${v}`);
|
|
287
|
-
console.log(` ${parts.join(' ')}`);
|
|
288
|
-
console.log('');
|
|
289
|
-
});
|
|
290
|
-
|
|
291
|
-
// Score histogram for the passed set so we see what the distribution looks like.
|
|
292
|
-
const buckets = { '5.0-5.9': 0, '6.0-6.9': 0, '7.0-7.9': 0, '8.0-8.9': 0, '9.0-10': 0 };
|
|
293
|
-
for (const r of passed) {
|
|
294
|
-
const s = r.grade.score;
|
|
295
|
-
if (s < 6) buckets['5.0-5.9']++;
|
|
296
|
-
else if (s < 7) buckets['6.0-6.9']++;
|
|
297
|
-
else if (s < 8) buckets['7.0-7.9']++;
|
|
298
|
-
else if (s < 9) buckets['8.0-8.9']++;
|
|
299
|
-
else buckets['9.0-10']++;
|
|
300
|
-
}
|
|
301
|
-
console.log('=== Passed-score histogram ===');
|
|
302
|
-
for (const [k, v] of Object.entries(buckets)) console.log(` ${k}: ${v}`);
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
// Run only as a CLI. Without this guard the file executes main() — and hits the
|
|
306
|
-
// API — the instant anything requires it, which is why its git helpers had never
|
|
307
|
-
// been tested: there was no way to reach them without starting an audit
|
|
308
|
-
// (task 1003223).
|
|
309
|
-
if (require.main === module) {
|
|
310
|
-
main().catch((err) => {
|
|
311
|
-
console.error(err);
|
|
312
|
-
process.exit(1);
|
|
313
|
-
});
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
// Exported for tests/audit_ships_git_argv.mjs — the argv-vs-shell-string fix is a
|
|
317
|
-
// property of THESE functions, so the test drives them rather than re-typing an
|
|
318
|
-
// equivalent git call and hoping it matches.
|
|
319
|
-
module.exports = { findCommitForTask, gitDiffStatsForCommit, filesTouchedByCommit, downstreamImpact };
|