@bongos/core 1.19.1036 → 1.19.1038
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +62 -52
- package/.claude/skills/grade-recover/SKILL.md +1 -1
- package/docs/adr/0089-modular-multi-model-adversarial-grader.md +2 -0
- package/docs/adr/0302-the-third-identical-failure-stops-the-session-and-asks.md +3 -2
- package/docs/adr/0329-the-rendered-fit-check-is-a-kit-floor-and-an-advisory-ship-check-row.md +2 -2
- package/docs/adr/README.md +1 -1
- package/docs/module-api-changelog.md +4 -0
- package/modules/economy/credits.js +1 -1
- package/modules/government/protected-surfaces.json +2 -0
- package/modules/grading/grader-config.js +3 -12
- package/modules/grading/grader-model.js +50 -5
- package/modules/grading/grader-no-artifact.js +227 -2
- package/modules/grading/grader-prompt.js +1 -1
- package/modules/grading/grader-providers.neutral.json +4 -0
- package/modules/grading/grader-rubric.json +6 -86
- package/modules/grading/grader-score.js +1 -169
- package/modules/grading/grader-subagent.js +18 -52
- package/modules/grading/grader.js +26 -211
- package/modules/grading/model-provider.js +257 -0
- package/modules/lifecycle/db-ship.js +3 -3
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +16 -0
- package/scripts/gds/claim.js +2 -1
- package/scripts/gds/grade-smoke.js +3 -3
- package/scripts/gds/render-check.js +1 -1
- package/scripts/gds/ship-check.js +1 -1
- package/scripts/gds/ship-flow.js +1 -1
- package/scripts/gds/ship-grade-steps.js +2 -2
- package/scripts/gds/visual-review-nudge.js +2 -1
- package/src/module-api.js +1 -1
- package/tests/adr_0162_gate_citation.mjs +167 -0
- package/tests/government_protected_surfaces.mjs +14 -0
- package/tests/grade_model_provider.mjs +217 -0
- package/tests/grader.mjs +48 -126
- package/tests/grader_unbacked_score.mjs +1 -1
- package/tests/regrade_actual_minutes.mjs +18 -17
- package/tests/render_check.mjs +1 -1
- package/tests/skill_grade_audit.mjs +3 -3
- package/tests/skill_grader_health.mjs +2 -2
- package/scripts/gds/audit-ships.js +0 -319
- package/tests/audit_ships_git_argv.mjs +0 -173
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
'use strict';
|
|
10
10
|
|
|
11
|
-
const { ASSUMED_BUILDER_MODEL, DEFAULT_GRADER_MODEL, WORKER_MODEL_TIERS, api } = require('./grader-config.js');
|
|
11
|
+
const { ASSUMED_BUILDER_MODEL, DEFAULT_GRADER_MODEL, V4, WORKER_MODEL_TIERS, api } = require('./grader-config.js');
|
|
12
12
|
|
|
13
13
|
function r1(n) {
|
|
14
14
|
return Math.round(Number(n) * 10) / 10;
|
|
@@ -93,12 +93,14 @@ function isTrivialDiff({ changedFiles, diffStats } = {}) {
|
|
|
93
93
|
// misconfiguration it guarded. Unknown ids now warn loudly and grade on
|
|
94
94
|
// DEFAULT_GRADER_MODEL; the id still reaches signals.builder_model_assumed,
|
|
95
95
|
// so /grader-health can spot the drift.
|
|
96
|
+
//
|
|
97
|
+
// task 1002840 (ADR 0089 §3): the Claude-only if/else is now DATA — the judge
|
|
98
|
+
// table below, first matching row wins, in the old branch order.
|
|
96
99
|
function selectGraderModel({ builderModel } = {}) {
|
|
97
100
|
const bm = (builderModel || ASSUMED_BUILDER_MODEL).toLowerCase();
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (bm.includes('haiku')) return 'claude-opus-5';
|
|
101
|
+
for (const row of JUDGE_TABLE) {
|
|
102
|
+
if (row.builder_families.some((f) => bm.includes(f))) return row.judge_model;
|
|
103
|
+
}
|
|
102
104
|
console.error(
|
|
103
105
|
`selectGraderModel: unrecognized builder model ${JSON.stringify(builderModel)} — ` +
|
|
104
106
|
`no 'fable'/'mythos'/'opus'/'sonnet'/'haiku' token; grading on the default ` +
|
|
@@ -112,6 +114,46 @@ function selectGraderModel({ builderModel } = {}) {
|
|
|
112
114
|
// back to graderModel) when it lands on the BUILDER's own family: tiering is
|
|
113
115
|
// a cost lever, never a hole in the ADR 0024 cross-model guarantee.
|
|
114
116
|
const MODEL_FAMILY_TOKENS = ['fable', 'mythos', 'opus', 'sonnet', 'haiku'];
|
|
117
|
+
|
|
118
|
+
// task 1002840: the judge table — which grader model each builder family gets.
|
|
119
|
+
// Read from the rubric (v4.judge_by_builder_family) and VALIDATED at load. A
|
|
120
|
+
// row is refused when its judge shares a family with a builder it serves (the
|
|
121
|
+
// ADR 0024 cross-model guarantee: no resolution may land the judge in the
|
|
122
|
+
// builder's own family) or is not a plain Claude model id — the GATING panel
|
|
123
|
+
// stays on Claude, the quality floor ADR 0089 keeps; an open model joins only as
|
|
124
|
+
// an advisory judge (task 1003858). A table that fails is replaced WHOLE by the
|
|
125
|
+
// built-in one below, which is the mapping the old if/else hard-coded, so a bad
|
|
126
|
+
// rubric edit degrades to today's behaviour instead of to a throw at grade time.
|
|
127
|
+
const BUILTIN_JUDGE_TABLE = Object.freeze([
|
|
128
|
+
Object.freeze({ builder_families: Object.freeze(['fable', 'mythos', 'opus']), judge_model: 'claude-sonnet-5' }),
|
|
129
|
+
Object.freeze({ builder_families: Object.freeze(['sonnet', 'haiku']), judge_model: 'claude-opus-5' }),
|
|
130
|
+
]);
|
|
131
|
+
|
|
132
|
+
function judgeTableProblems(table) {
|
|
133
|
+
if (!Array.isArray(table) || table.length === 0) return ['judge_by_builder_family must be a non-empty array'];
|
|
134
|
+
const problems = [];
|
|
135
|
+
table.forEach((row, i) => {
|
|
136
|
+
const fams = row && Array.isArray(row.builder_families) ? row.builder_families : null;
|
|
137
|
+
const judge = row && typeof row.judge_model === 'string' ? row.judge_model.toLowerCase() : null;
|
|
138
|
+
if (!fams || fams.length === 0 || !fams.every((f) => MODEL_FAMILY_TOKENS.includes(f))) {
|
|
139
|
+
problems.push(`row ${i}: builder_families must list family tokens (${MODEL_FAMILY_TOKENS.join('/')})`);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const judgeFamily = judge && !judge.includes(':') ? MODEL_FAMILY_TOKENS.find((t) => judge.includes(t)) : null;
|
|
143
|
+
if (!judgeFamily) problems.push(`row ${i}: judge_model must be a plain Claude model id (the gating panel stays on Claude)`);
|
|
144
|
+
else if (fams.includes(judgeFamily)) problems.push(`row ${i}: judge '${row.judge_model}' is in the '${judgeFamily}' family it would grade (ADR 0024)`);
|
|
145
|
+
});
|
|
146
|
+
return problems;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function resolveJudgeTable(table) {
|
|
150
|
+
const problems = judgeTableProblems(table);
|
|
151
|
+
if (problems.length === 0) return table;
|
|
152
|
+
console.error(`grader: rubric v4.judge_by_builder_family refused (${problems.join('; ')}) — grading on the built-in judge table`);
|
|
153
|
+
return BUILTIN_JUDGE_TABLE;
|
|
154
|
+
}
|
|
155
|
+
const JUDGE_TABLE = resolveJudgeTable(V4.judge_by_builder_family);
|
|
156
|
+
|
|
115
157
|
function workerModelFor(kind, graderModel, builderModel) {
|
|
116
158
|
const tier = WORKER_MODEL_TIERS[kind];
|
|
117
159
|
if (!tier) return graderModel;
|
|
@@ -137,12 +179,15 @@ const LLM_PROJECT = (() => {
|
|
|
137
179
|
|
|
138
180
|
|
|
139
181
|
module.exports = {
|
|
182
|
+
BUILTIN_JUDGE_TABLE,
|
|
183
|
+
JUDGE_TABLE,
|
|
140
184
|
LLM_PROJECT,
|
|
141
185
|
MODEL_FAMILY_TOKENS,
|
|
142
186
|
TRIVIAL_MAX_FILES,
|
|
143
187
|
TRIVIAL_MAX_LINES,
|
|
144
188
|
clamp,
|
|
145
189
|
isTrivialDiff,
|
|
190
|
+
judgeTableProblems,
|
|
146
191
|
median,
|
|
147
192
|
r1,
|
|
148
193
|
selectGraderModel,
|
|
@@ -47,8 +47,9 @@
|
|
|
47
47
|
|
|
48
48
|
const { LLM_PROJECT, selectGraderModel } = require('./grader-model.js');
|
|
49
49
|
const { ASSUMED_BUILDER_MODEL, THRESHOLD, VERSION } = require('./grader-config.js');
|
|
50
|
-
const {
|
|
51
|
-
const {
|
|
50
|
+
const { extractAssistantResult, runSubagent } = require('./grader-subagent.js');
|
|
51
|
+
const { validateAndScore } = require('./grader-score.js');
|
|
52
|
+
const { median, r1 } = require('./grader-model.js');
|
|
52
53
|
// SR-15 / #995. Every input this prompt carries is builder-controlled, and this
|
|
53
54
|
// species is the HIGHEST-stakes place in the grader to leave that unfenced — see the
|
|
54
55
|
// note above buildNoArtifactPrompt.
|
|
@@ -56,6 +57,228 @@ const { NOFOLLOW_PREAMBLE, fenceBlock } = require('./grader-workers/fence.js');
|
|
|
56
57
|
|
|
57
58
|
const GRADER_KIND = 'subagent-no-artifact';
|
|
58
59
|
|
|
60
|
+
// ---------- the generalist shot: one prompt, N independent readings ----------
|
|
61
|
+
//
|
|
62
|
+
// task 1002842 moved these two here from grader-subagent.js / grader-score.js. They
|
|
63
|
+
// began as the v3 multi-shot grader's plumbing; the v3 GENERATION is retired, but
|
|
64
|
+
// this species (and the trivial-diff fast path in grader.js, which still runs one
|
|
65
|
+
// generalist shot) is built on them, so they live with their newest caller rather
|
|
66
|
+
// than as a leftover of the generation that introduced them. Behaviour unchanged —
|
|
67
|
+
// tests/grader.mjs pins aggregateShots and this file's tests pin the wiring.
|
|
68
|
+
|
|
69
|
+
// Run one shot end-to-end: build the prompt (caller did this), spawn the
|
|
70
|
+
// subagent at the requested model, parse + validate. Returns a per-shot
|
|
71
|
+
// grade-result; subprocess failures become passed=false per-shot results so
|
|
72
|
+
// aggregateShots can still produce a final verdict.
|
|
73
|
+
async function runOneShot({ prompt, model, shotIndex, runOpts }) {
|
|
74
|
+
try {
|
|
75
|
+
const { stdout, run_ms, exit_code, cost_usd } = await runSubagent({
|
|
76
|
+
prompt,
|
|
77
|
+
opts: { ...runOpts, model },
|
|
78
|
+
});
|
|
79
|
+
const assistantResult = extractAssistantResult(stdout);
|
|
80
|
+
const scored = validateAndScore(assistantResult, {
|
|
81
|
+
run_ms,
|
|
82
|
+
exit_code,
|
|
83
|
+
raw_chars: stdout.length,
|
|
84
|
+
model,
|
|
85
|
+
shot_index: shotIndex,
|
|
86
|
+
});
|
|
87
|
+
// #486 (ADR 0028 path 1): carry this shot's own subprocess cost so the
|
|
88
|
+
// trivial-diff path (gradeSubagentV3MultiShot → aggregateShots) can sum it
|
|
89
|
+
// into panel_cost_usd just like the full panel does.
|
|
90
|
+
scored.cost_usd = cost_usd ?? null;
|
|
91
|
+
return scored;
|
|
92
|
+
} catch (err) {
|
|
93
|
+
return {
|
|
94
|
+
score: 0,
|
|
95
|
+
passed: false,
|
|
96
|
+
threshold: THRESHOLD,
|
|
97
|
+
grader_kind: 'subagent',
|
|
98
|
+
grader_version: VERSION,
|
|
99
|
+
rubric_scores: {},
|
|
100
|
+
signals: {
|
|
101
|
+
grader_kind: 'subagent',
|
|
102
|
+
grader_version: VERSION,
|
|
103
|
+
model,
|
|
104
|
+
threshold: THRESHOLD,
|
|
105
|
+
shot_index: shotIndex,
|
|
106
|
+
run_ms: err.run_ms ?? null,
|
|
107
|
+
exit_code: err.exit_code ?? null,
|
|
108
|
+
subprocess_error: err.code || 'UNKNOWN',
|
|
109
|
+
stderr_tail: (err.stderr || '').slice(-500),
|
|
110
|
+
},
|
|
111
|
+
notes_md: `Shot ${shotIndex}: subprocess failed (${err.code || 'UNKNOWN'}: ${err.message}).`,
|
|
112
|
+
issues: [],
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Collapses N per-shot grade-results into the final v3 grade-result.
|
|
118
|
+
// Pass requires unanimous PASS on all shots; anything else (any FAIL, any
|
|
119
|
+
// error, any disagreement) → not passed. Issues are deduped across shots
|
|
120
|
+
// by (file, summary). Displayed rubric_scores use the median across shots
|
|
121
|
+
// so the trend dashboard stays one-number-per-task.
|
|
122
|
+
function aggregateShots(shotResults, meta = {}) {
|
|
123
|
+
const N = shotResults.length;
|
|
124
|
+
const verdicts = shotResults.map((s) => ({
|
|
125
|
+
passed: s.passed,
|
|
126
|
+
ff: s.rubric_scores?.functional_fidelity ?? null,
|
|
127
|
+
cq: s.rubric_scores?.code_quality ?? null,
|
|
128
|
+
economy: s.rubric_scores?.economy ?? null,
|
|
129
|
+
cq_effective: s.rubric_scores?.code_quality_effective ?? null,
|
|
130
|
+
confidence: s.signals?.confidence ?? null,
|
|
131
|
+
model: s.signals?.model ?? null,
|
|
132
|
+
run_ms: s.signals?.run_ms ?? null,
|
|
133
|
+
shot_index: s.signals?.shot_index ?? null,
|
|
134
|
+
notes_md: s.notes_md,
|
|
135
|
+
issues: s.issues || [],
|
|
136
|
+
error:
|
|
137
|
+
s.signals?.parse_error
|
|
138
|
+
? 'parse_error'
|
|
139
|
+
: s.signals?.schema_error
|
|
140
|
+
? 'schema_error'
|
|
141
|
+
: s.signals?.subprocess_error || null,
|
|
142
|
+
}));
|
|
143
|
+
|
|
144
|
+
const ff = median(verdicts.map((v) => v.ff));
|
|
145
|
+
const cq = median(verdicts.map((v) => v.cq));
|
|
146
|
+
const economy = median(verdicts.map((v) => v.economy));
|
|
147
|
+
const cqEffective = median(verdicts.map((v) => v.cq_effective));
|
|
148
|
+
// NB (task 1003318): this v3 multi-shot path still collapses a null axis to
|
|
149
|
+
// 0 — it serves the trivial-diff single shot, which cannot overflow a window;
|
|
150
|
+
// the panel path's lens-outage handling lives in validateAndScore.
|
|
151
|
+
const blended = ff != null && cq != null ? r1((ff + cq) / 2) : 0;
|
|
152
|
+
// Top-level aggregated confidence = median of per-shot confidences. Without
|
|
153
|
+
// this, ship.js forwards a null top-level confidence to /tasks/:id/grade,
|
|
154
|
+
// and the server's validateAndScore (re-running on the POST body) hits the
|
|
155
|
+
// Number(null)=0 trap → FAIL even though every shot independently passed.
|
|
156
|
+
// Caught on the MAS Phase 1 self-ship (#395).
|
|
157
|
+
//
|
|
158
|
+
// Computed UNROUNDED (no r1()) — confidence sits near a 0.4 gate cap so
|
|
159
|
+
// rounding to one decimal could nudge a borderline value across the cap
|
|
160
|
+
// in either direction. Per-shot confidences are already 0-1 floats.
|
|
161
|
+
const confVals = verdicts
|
|
162
|
+
.map((v) => v.confidence)
|
|
163
|
+
.filter((n) => Number.isFinite(n))
|
|
164
|
+
.sort((a, b) => a - b);
|
|
165
|
+
const confidenceMedian = confVals.length === 0
|
|
166
|
+
? null
|
|
167
|
+
: confVals.length % 2
|
|
168
|
+
? confVals[(confVals.length - 1) / 2]
|
|
169
|
+
: (confVals[confVals.length / 2 - 1] + confVals[confVals.length / 2]) / 2;
|
|
170
|
+
|
|
171
|
+
// Dedupe issues across shots by (file, summary). Keep first-seen severity.
|
|
172
|
+
const seen = new Set();
|
|
173
|
+
const issues = [];
|
|
174
|
+
for (const v of verdicts) {
|
|
175
|
+
for (const it of v.issues) {
|
|
176
|
+
if (!it || typeof it !== 'object') continue;
|
|
177
|
+
const key = `${it.file ?? ''}::${(it.summary ?? '').trim()}`;
|
|
178
|
+
if (seen.has(key)) continue;
|
|
179
|
+
seen.add(key);
|
|
180
|
+
issues.push(it);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const passCount = verdicts.filter((v) => v.passed === true).length;
|
|
185
|
+
const failCount = verdicts.filter((v) => v.passed === false && !v.error).length;
|
|
186
|
+
const errorCount = verdicts.filter((v) => v.error).length;
|
|
187
|
+
const allPassed = N > 0 && passCount === N;
|
|
188
|
+
|
|
189
|
+
let verdict;
|
|
190
|
+
let passed;
|
|
191
|
+
let aggregationKind;
|
|
192
|
+
if (errorCount > 0) {
|
|
193
|
+
passed = false;
|
|
194
|
+
verdict = `FAIL — ${errorCount}/${N} shot(s) errored`;
|
|
195
|
+
aggregationKind = 'errored';
|
|
196
|
+
} else if (allPassed) {
|
|
197
|
+
passed = true;
|
|
198
|
+
verdict = `PASS — ${N}/${N} shots unanimous`;
|
|
199
|
+
aggregationKind = 'unanimous_pass';
|
|
200
|
+
} else if (passCount === 0) {
|
|
201
|
+
passed = false;
|
|
202
|
+
verdict = `FAIL — ${failCount}/${N} shots`;
|
|
203
|
+
aggregationKind = 'unanimous_fail';
|
|
204
|
+
} else {
|
|
205
|
+
passed = false;
|
|
206
|
+
verdict = `FAIL — ${passCount} pass, ${failCount} fail (no unanimous pass)`;
|
|
207
|
+
aggregationKind = 'disagreement';
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const lines = [];
|
|
211
|
+
lines.push(`Subagent grader ${VERSION} (multi-shot, ${N}× shots) — ${verdict}`);
|
|
212
|
+
lines.push(` functional_fidelity (median): ${ff ?? '?'}/10`);
|
|
213
|
+
lines.push(` code_quality (median): ${cq ?? '?'}/10`);
|
|
214
|
+
if (economy != null) {
|
|
215
|
+
lines.push(` economy (advisory, median): ${economy}/10`);
|
|
216
|
+
}
|
|
217
|
+
lines.push(` blended (median): ${blended}/10 (threshold ${THRESHOLD}, both axes must clear on EVERY shot)`);
|
|
218
|
+
lines.push('');
|
|
219
|
+
lines.push('Per-shot:');
|
|
220
|
+
for (let i = 0; i < verdicts.length; i++) {
|
|
221
|
+
const v = verdicts[i];
|
|
222
|
+
const status = v.error ? 'ERROR' : v.passed === true ? 'PASS' : 'FAIL';
|
|
223
|
+
const ffStr = v.ff != null ? `${v.ff}` : '?';
|
|
224
|
+
const cqStr = v.cq != null ? `${v.cq}` : '?';
|
|
225
|
+
const confStr = v.confidence != null ? ` conf=${v.confidence}` : '';
|
|
226
|
+
const errStr = v.error ? ` (${v.error})` : '';
|
|
227
|
+
lines.push(` shot[${i}] ${status} ff=${ffStr} cq=${cqStr}${confStr} ${v.model ?? '?'}${errStr}`);
|
|
228
|
+
}
|
|
229
|
+
if (issues.length > 0) {
|
|
230
|
+
lines.push('');
|
|
231
|
+
lines.push(`Issues (${issues.length} unique across shots):`);
|
|
232
|
+
for (const it of issues.slice(0, 20)) {
|
|
233
|
+
const sev = String(it.severity || '?').padEnd(7);
|
|
234
|
+
const file = String(it.file || '?');
|
|
235
|
+
const summary = String(it.summary || '').replace(/\s+/g, ' ').trim();
|
|
236
|
+
lines.push(` [${sev}] ${file} — ${summary}`);
|
|
237
|
+
}
|
|
238
|
+
if (issues.length > 20) {
|
|
239
|
+
lines.push(` ... ${issues.length - 20} more (full set in signals.shots[].issues)`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// #486 (ADR 0028 path 1): sum each shot's own subprocess cost (envelope
|
|
244
|
+
// total_cost_usd) so the trivial-diff path reports panel_cost_usd the same way
|
|
245
|
+
// the full panel does. The single-shot trivial path is the common case here.
|
|
246
|
+
const panelCostUsd = shotResults.reduce((s, r) => s + (Number(r.cost_usd) || 0), 0);
|
|
247
|
+
|
|
248
|
+
return {
|
|
249
|
+
score: blended,
|
|
250
|
+
passed,
|
|
251
|
+
threshold: THRESHOLD,
|
|
252
|
+
grader_kind: 'subagent-multishot',
|
|
253
|
+
grader_version: VERSION,
|
|
254
|
+
panel_cost_usd: panelCostUsd, // #486: panel's own API spend (USD)
|
|
255
|
+
rubric_scores: {
|
|
256
|
+
functional_fidelity: ff,
|
|
257
|
+
code_quality: cq,
|
|
258
|
+
economy,
|
|
259
|
+
code_quality_effective: cqEffective,
|
|
260
|
+
},
|
|
261
|
+
signals: {
|
|
262
|
+
grader_kind: 'subagent-multishot',
|
|
263
|
+
grader_version: VERSION,
|
|
264
|
+
threshold: THRESHOLD,
|
|
265
|
+
// Top-level confidence = median across shots. ship.js forwards this to
|
|
266
|
+
// POST /tasks/:id/grade so the server-side re-validation has a real
|
|
267
|
+
// number rather than null (see Number(null)=0 fix in validateAndScore).
|
|
268
|
+
confidence: confidenceMedian,
|
|
269
|
+
shots: verdicts,
|
|
270
|
+
shot_count: N,
|
|
271
|
+
pass_count: passCount,
|
|
272
|
+
fail_count: failCount,
|
|
273
|
+
error_count: errorCount,
|
|
274
|
+
shot_aggregation: aggregationKind,
|
|
275
|
+
...meta,
|
|
276
|
+
},
|
|
277
|
+
notes_md: lines.join('\n'),
|
|
278
|
+
issues,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
59
282
|
// How much of the builder's prose the grader is shown. Handoff notes are the
|
|
60
283
|
// EVIDENCE here rather than context around it, so the cap is generous — but it is a
|
|
61
284
|
// cap, because an unbounded field reaches the model as an unbounded prompt.
|
|
@@ -339,9 +562,11 @@ async function gradeNoArtifact({ task, valueSummary, notes, builderModel, runOpt
|
|
|
339
562
|
// which is the thing worth asserting on anyway.
|
|
340
563
|
module.exports = {
|
|
341
564
|
GRADER_KIND,
|
|
565
|
+
aggregateShots,
|
|
342
566
|
NOTES_CHAR_CAP,
|
|
343
567
|
SHOTS,
|
|
344
568
|
buildNoArtifactPrompt,
|
|
345
569
|
collapseUnanimous,
|
|
346
570
|
gradeNoArtifact,
|
|
571
|
+
runOneShot,
|
|
347
572
|
};
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
'use strict';
|
|
10
10
|
|
|
11
11
|
const path = require('node:path');
|
|
12
|
-
const {
|
|
12
|
+
const { generatedProvenance, workers } = require('./grader-config.js');
|
|
13
13
|
const { LLM_PROJECT } = require('./grader-model.js');
|
|
14
14
|
// The written role responsibilities, through the doorway (task 1003732). Lazy on
|
|
15
15
|
// the api object, so requiring it here costs nothing until a prompt is built.
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_comment": "The core's neutral starter for grader model providers (task 1002840, ADR 0089 §3): NONE, so a vanilla instance grades on Claude alone. An instance declares its own in config/grader-providers.json at the instance root. Each entry is { kind: 'openai-compatible', base_url: https://…, api_key_env: <ENV VAR NAME, never the key>, price_per_mtok: { <model>: { input, output } } }. Declaring one is an owner decision: it sends the diff under review to that vendor and costs money per call.",
|
|
3
|
+
"providers": {}
|
|
4
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"current_version": "v4",
|
|
3
3
|
"threshold": 6.0,
|
|
4
4
|
"version": "v4.0",
|
|
5
|
-
"_top_level_alias_note": "`threshold` and `version` mirror the current_version's threshold + version so consumers that still read grader.RUBRIC.threshold (e.g. routes/public.js
|
|
5
|
+
"_top_level_alias_note": "`threshold` and `version` mirror the current_version's threshold + version so consumers that still read grader.RUBRIC.threshold (e.g. routes/public.js) keep working. Update all three (current_version, top-level threshold, top-level version) when promoting a new grader version.",
|
|
6
6
|
"v4": {
|
|
7
7
|
"version": "v4.0",
|
|
8
8
|
"grader_kind": "subagent-panel",
|
|
@@ -14,6 +14,11 @@
|
|
|
14
14
|
"active_panel": ["narc", "quality", "hacker", "efficiency"],
|
|
15
15
|
"worker_model_tiers": { "quality": "claude-sonnet-5", "reader": "claude-sonnet-5" },
|
|
16
16
|
"worker_model_tiers_note": "task 1002660: per-worker cost tiering. Quality was 41% of panel spend with 75% nit findings — its cq/economy lens ran on the haiku tier. task 1003318: moved to claude-sonnet-5 — haiku 4.5's 200k window overflows on any diff over ~150 KB (`Prompt is too long`, deterministic; tasks 1003297 and 1003298 each lost three ~$5 panel runs to it) and a Quality worker that never starts is missing signal on a GATING lens. For fable/opus builders the panel model is already claude-sonnet-5, so the tier coincides with default_model and acts as a floor — a future default_model bump must not pin Quality back onto a smaller window; it stays a live cost lever only for haiku builders (whose panel runs on opus-5). workerModelFor refuses an override that lands on the builder's own model family (the ADR 0024 cross-model guarantee outranks the cost lever); gate semantics unchanged — Quality's verdict still gates identically. task 1003977: the READER is pinned to claude-sonnet-5 for the same task-1003318 reason — it is Read-heavy by construction (the voice guide, then the render paths that decide whether a string is user-facing), and a small window fails deterministically with PROMPT_TOO_LONG. For a NON-GATING worker that means silently reading no copy at all on exactly the biggest copy diffs, with nothing blocking to reveal it.",
|
|
17
|
+
"judge_by_builder_family": [
|
|
18
|
+
{ "builder_families": ["fable", "mythos", "opus"], "judge_model": "claude-sonnet-5" },
|
|
19
|
+
{ "builder_families": ["sonnet", "haiku"], "judge_model": "claude-opus-5" }
|
|
20
|
+
],
|
|
21
|
+
"judge_by_builder_family_note": "task 1002840 (ADR 0089 §3): the cross-family judge choice selectGraderModel used to hard-code as an if/else — the first row whose family token appears in the builder's model id wins. Validated at load (grader-model.js judgeTableProblems): a judge in the family it grades (ADR 0024) or anything but a plain Claude id is refused and the whole table falls back to the built-in copy of exactly these two rows. An unmatched builder still grades on default_model with a warning (task 1002659).",
|
|
17
22
|
"fallthrough_confidence": 0.3,
|
|
18
23
|
"low_confidence_threshold": 0.4,
|
|
19
24
|
"rationale": "GDS-V3 task #410 (MAS Phase 2.0), 2026-05-29. Supersedes v3 (single-prompt 3-shot generalist). Phase 1 calibration showed minimal between-shot disagreement on the same prompt — the strict gate fired correctly but the diversity signal was weak. v4 trades sampling-variance for lens-specialization: replace 3 identical-prompt shots with N workers (each a distinct prompt focused on one axis). Phase 2.0 ships 2 of the 3 planned workers (narc for completion, quality for cq + advisory economy) — keeps the gate at the same strictness while the framework calibrates. Phase 2.1 adds hacker (security; wraps /builder-redteam as a CI gate). Phase 2.2 adds efficiency_monger.",
|
|
@@ -76,90 +81,5 @@
|
|
|
76
81
|
},
|
|
77
82
|
"rubric_scores_audit_trail": "task_grades.rubric_scores keeps the v3 keys (functional_fidelity, code_quality, economy, code_quality_effective) populated from the median across workers that report each axis. Narc reports ff; Quality reports cq + economy. Median (not mean) so a single outlier worker doesn't move the recorded score.",
|
|
78
83
|
"server_compat_note": "The grade POST body (POST /api/bongos/tasks/:id/grade) keeps the v3 shape (functional_fidelity, code_quality, economy, confidence, notes_md, issues) so the server's existing validateAndScore re-validation works without modification. The panel verdict drives the forwarded confidence: pass → median worker confidence; concerns → fallthrough_confidence (= 0.3); fail → 0. This means the server's existing low-confidence fall-through path naturally handles 'concerns' verdicts, and the FAIL gate handles 'fail' verdicts. Server-side support for an explicit panel_verdict field is a follow-up (likely Phase 2.5)."
|
|
79
|
-
},
|
|
80
|
-
"v3": {
|
|
81
|
-
"version": "v3.0",
|
|
82
|
-
"grader_kind": "subagent-multishot",
|
|
83
|
-
"threshold": 6.0,
|
|
84
|
-
"status": "DEPRECATED — superseded by v4 in GDS-V3 task #410 (2026-05-29). The 3-shot single-prompt path is kept reachable via grader.gradeSubagentV3MultiShot for explicit single-generalist-prompt diagnostics, but is not the active gate.",
|
|
85
|
-
"gate": "Both axes (functional_fidelity AND code_quality) must clear threshold on EVERY shot (3-shot unanimous-PASS gate) for confirmed→credits. Any single FAIL, any disagreement between shots, or any low-confidence shot keeps the task at completed; builder can manual-confirm via POST /tasks/:id/confirm if they think the grade is wrong (calibration release-valve — see ADR 0024).",
|
|
86
|
-
"default_model": "claude-sonnet-4-6",
|
|
87
|
-
"assumed_builder_model": "claude-opus-4-7",
|
|
88
|
-
"shots": 3,
|
|
89
|
-
"shot_aggregation": "unanimous_pass",
|
|
90
|
-
"rationale": "GDS-V3 task #395 (MAS Phase 1), 2026-05-29. Supersedes v2 (single-shot Opus on assumed-Opus builder, default-pass framing) which suffered from three structural leniencies: (1) same-family judge inflation ~10-15% per Zheng et al. 2023 and Chiang & Lee 2023; (2) no disagreement signal — one inflated score = pass; (3) default-pass prompt framing biased the grader toward 'find a score' over 'find what's wrong'. v3 fixes all three: (1) switches the default grader model to Sonnet 4.6 (cross-family from assumed-Opus builder); (2) runs 3 independent shots and requires unanimous PASS; (3) rewrites the prompt to default-fail (issues enumerated first, pass only if no material problems remain). See ADR 0024 (multi-agent system architecture) for full context and the calibration plan.",
|
|
91
|
-
"rubric_axes": {
|
|
92
|
-
"functional_fidelity": {
|
|
93
|
-
"scale": "0-10",
|
|
94
|
-
"doc": "Did the diff implement what the description scoped? Scope gaps, half-implementations, hardcoded shortcuts, TODOs left in, dummy/demo code that lies — all drag this down. 10 = every acceptance criterion in the description is satisfied by code (or explicitly out-of-scope per the description). 6 = mostly there with one named gap. 0 = the diff does not implement the described work or implements something else."
|
|
95
|
-
},
|
|
96
|
-
"code_quality": {
|
|
97
|
-
"scale": "0-10",
|
|
98
|
-
"doc": "Idiomatic for THIS codebase, readable, tested where new logic warrants tests, no dead code introduced, doesn't break invariants (e.g. methodology rules in CLAUDE.md, cross-verified Python/JS autotile lookups, ground-vs-object cell split per ADR 0012, the 'every change is backed by a claimed GDS task' rule, etc.). 10 = production-quality, would-merge-without-comment. 6 = ships but with a clear improvement the reviewer would suggest. 0 = obviously broken, hostile to maintenance."
|
|
99
|
-
},
|
|
100
|
-
"economy": {
|
|
101
|
-
"scale": "0-10",
|
|
102
|
-
"doc": "ADVISORY (carried from v2 / ADR 0017). Economy of the change: is the diff as small and simple as the task warrants? Penalize unnecessary abstraction, premature generalization, duplicated or dead code, and diffs larger than the work needs. Advisory: recorded in rubric_scores and blended into an advisory code_quality_effective signal, but does NOT independently hard-block. Promotion to a third both-must-pass axis remains a follow-up after calibration."
|
|
103
|
-
}
|
|
104
|
-
},
|
|
105
|
-
"shot_aggregation_doc": "v3 runs 3 independent grader subagents in parallel against the same prompt. Pass requires ALL THREE shots to report PASS (functional_fidelity >= threshold AND code_quality >= threshold AND confidence >= low_confidence_threshold). Any FAIL, any disagreement, or any low-confidence shot → fall through to completed state and existing manual-confirm endpoint. Per-shot results are persisted in task_grades.signals.shots[]; displayed rubric_scores use the median across shots (so the trend dashboard stays one-number-per-task). The blended task_grades.score is the median (ff+cq)/2 across shots.",
|
|
106
|
-
"cross_family_doc": "v3's default grader model (Sonnet 4.6) is intentionally different from the assumed builder model (Opus 4.7). This carves out from CLAUDE.md §2's 'default to Opus' rule — that rule covers subagents that produce work that ships (research → code); grading is *verification*, where adversarial cross-family perspective beats raw capability. When per-builder model preferences land (ADR 0021 follow-up), the grader's selectGraderModel(builderModel) function generalizes this: builder Opus → grader Sonnet; builder Sonnet → grader Opus; builder Haiku → grader Opus. The panel composition (3× Sonnet) was chosen over a mixed panel (Sonnet/Opus/Haiku) to keep grading-scale consistent across shots; if 3× Sonnet shows insufficient disagreement signal during calibration, Phase 1.5 mixes the panel.",
|
|
107
|
-
"economy_gate_note": "Carried over from v2 (V3.R92 #353). Economy remains ADVISORY in v3. The hard gate stays functional_fidelity >= threshold AND code_quality >= threshold, each on its own raw score, unanimous across shots. Economy nudges code_quality_effective DOWN only, never gates by itself. Hardening economy into a third both-must-pass axis remains deferred until calibration data exists.",
|
|
108
|
-
"economy_blend": "code_quality_effective = code_quality - max(0, (THRESHOLD - economy) / 2), clamped to [0,10]. Applied per-shot; the displayed median uses median(code_quality_effective) for the trend dashboard. Not used by the pass gate.",
|
|
109
|
-
"output_schema": {
|
|
110
|
-
"type": "object",
|
|
111
|
-
"required": ["functional_fidelity", "code_quality", "notes_md", "issues", "confidence"],
|
|
112
|
-
"properties": {
|
|
113
|
-
"functional_fidelity": { "type": "number", "minimum": 0, "maximum": 10 },
|
|
114
|
-
"code_quality": { "type": "number", "minimum": 0, "maximum": 10 },
|
|
115
|
-
"economy": { "type": "number", "minimum": 0, "maximum": 10, "doc": "ADVISORY. Recorded and blended into code_quality_effective; does NOT hard-block. Optional for back-compat: absent → recorded as null, no penalty." },
|
|
116
|
-
"security_score": { "type": "number", "minimum": 0, "maximum": 10, "doc": "ADVISORY (MAS Phase 2.5 #423). Reported by Hacker (Phase 2.1). Recorded in rubric_scores for audit + trend; does NOT hard-block server-side — Hacker's per-worker verdict drives the panel gate via confidence-forwarding. Optional for back-compat: absent → recorded as null, no penalty." },
|
|
117
|
-
"efficiency_score": { "type": "number", "minimum": 0, "maximum": 10, "doc": "ADVISORY (MAS Phase 2.5 #423). Reported by Efficiency Monger (Phase 2.2). Recorded in rubric_scores for audit + trend; does NOT hard-block server-side — Efficiency Monger's per-worker verdict drives the panel gate via confidence-forwarding. Optional for back-compat: absent → recorded as null, no penalty." },
|
|
118
|
-
"notes_md": { "type": "string" },
|
|
119
|
-
"confidence": { "type": "number", "minimum": 0, "maximum": 1, "doc": "Self-reported confidence (0-1). Low confidence (<0.4) on ANY shot causes a fall-through to manual /tasks/:id/confirm." },
|
|
120
|
-
"issues": {
|
|
121
|
-
"type": "array",
|
|
122
|
-
"items": {
|
|
123
|
-
"type": "object",
|
|
124
|
-
"required": ["severity", "file", "summary"],
|
|
125
|
-
"properties": {
|
|
126
|
-
"severity": { "type": "string", "enum": ["blocker", "major", "minor", "nit"] },
|
|
127
|
-
"file": { "type": "string" },
|
|
128
|
-
"summary": { "type": "string" }
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
},
|
|
134
|
-
"low_confidence_threshold": 0.4,
|
|
135
|
-
"blended_score_for_audit_trail": "We blend the two axes 50/50 (median across shots) into the task_grades.score column so the existing trend dashboard and view (task_grades_with_context) keep working. The gate, though, requires BOTH axes >= threshold ON EVERY SHOT — a median of 7.5 across shots does NOT pass if any individual shot was below 6.0."
|
|
136
|
-
},
|
|
137
|
-
"v1": {
|
|
138
|
-
"version": "v1.1",
|
|
139
|
-
"grader_kind": "auto",
|
|
140
|
-
"threshold": 6.0,
|
|
141
|
-
"status": "DEPRECATED — superseded by v2 in GDS-V3 task #209 (2026-05-13). The v1 entry points still exist in modules/grading/grader.js (gradeDeterministic + per-criterion helpers) because scripts/gds/audit-ships.js (the retroactive #204 audit) imports them. v1 is no longer invoked at ship time.",
|
|
142
|
-
"rationale": "Raised from 5.0 → 6.0 on 2026-05-12 (PMS-V2 #207) after the retroactive audit of #204 graded 92 ships and found 0 failures, median 6.2, floor 5.2. At threshold 5.0 the gate was structurally incapable of failing a shipped task because value_summary + scanner_smoke alone delivered ~5.2 even with everything else zeroed.",
|
|
143
|
-
"weights": {
|
|
144
|
-
"common": {
|
|
145
|
-
"est_actual_band": { "weight": 0.25, "doc": "actual_minutes / est_minutes ratio. 10 in [0.5, 2.0], drops linearly outside, 0 below 0.1 or above 10." },
|
|
146
|
-
"value_summary": { "weight": 0.25, "doc": "0 if empty, 5 if <40 chars, 8 if 40-120 chars, 10 if 120+ chars and contains a verb." },
|
|
147
|
-
"diff_hygiene": { "weight": 0.20, "doc": "10 if 1-50 files changed and lines_added+lines_removed > 0. 5 if files_changed > 50 (too broad). 0 if zero diff (phantom ship)." },
|
|
148
|
-
"scanner_smoke": { "weight": 0.10, "doc": "Constant 10 at grader time (already hard-gated upstream). Recorded for audit trail." },
|
|
149
|
-
"scoping_signal": { "weight": 0.20, "doc": "Reads tasks.scoping_confidence: 0=2/10, 1=4, 2=6, 3=7, 4=8, 5=10. NULL=6 (neutral)." }
|
|
150
|
-
},
|
|
151
|
-
"kind_specific": {
|
|
152
|
-
"cleanup": {
|
|
153
|
-
"loc_net_negative": { "weight": 0.20, "doc": "Bonus criterion that REPLACES diff_hygiene weight for cleanup tasks. 10 if lines_removed > lines_added. 5 if net-zero. 0 if cleanup increased LOC." }
|
|
154
|
-
},
|
|
155
|
-
"learning-capture": {
|
|
156
|
-
"learning_row": { "weight": 0.20, "doc": "Bonus criterion that REPLACES diff_hygiene for learning-capture. 10 if a learnings row exists for the task and body_md > 100 chars. 5 if row exists but body short. 0 if no row." }
|
|
157
|
-
},
|
|
158
|
-
"art-pipeline": {
|
|
159
|
-
"art_rubric_avg": { "weight": 0.30, "doc": "Bonus criterion. Average of art rubric scores from local art/iterations/<task>/summary.json. Passed up by ship.js in signals.art_rubric_avg. Skipped if absent." }
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
},
|
|
163
|
-
"shape_note": "Common weights sum to 1.0. Kind-specific bonus weights are merged in by REPLACING the matching common weight name where applicable; otherwise additive then renormalized so total stays 1.0."
|
|
164
84
|
}
|
|
165
85
|
}
|