@bongos/core 1.19.1036 → 1.19.1038
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +62 -52
- package/.claude/skills/grade-recover/SKILL.md +1 -1
- package/docs/adr/0089-modular-multi-model-adversarial-grader.md +2 -0
- package/docs/adr/0302-the-third-identical-failure-stops-the-session-and-asks.md +3 -2
- package/docs/adr/0329-the-rendered-fit-check-is-a-kit-floor-and-an-advisory-ship-check-row.md +2 -2
- package/docs/adr/README.md +1 -1
- package/docs/module-api-changelog.md +4 -0
- package/modules/economy/credits.js +1 -1
- package/modules/government/protected-surfaces.json +2 -0
- package/modules/grading/grader-config.js +3 -12
- package/modules/grading/grader-model.js +50 -5
- package/modules/grading/grader-no-artifact.js +227 -2
- package/modules/grading/grader-prompt.js +1 -1
- package/modules/grading/grader-providers.neutral.json +4 -0
- package/modules/grading/grader-rubric.json +6 -86
- package/modules/grading/grader-score.js +1 -169
- package/modules/grading/grader-subagent.js +18 -52
- package/modules/grading/grader.js +26 -211
- package/modules/grading/model-provider.js +257 -0
- package/modules/lifecycle/db-ship.js +3 -3
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +16 -0
- package/scripts/gds/claim.js +2 -1
- package/scripts/gds/grade-smoke.js +3 -3
- package/scripts/gds/render-check.js +1 -1
- package/scripts/gds/ship-check.js +1 -1
- package/scripts/gds/ship-flow.js +1 -1
- package/scripts/gds/ship-grade-steps.js +2 -2
- package/scripts/gds/visual-review-nudge.js +2 -1
- package/src/module-api.js +1 -1
- package/tests/adr_0162_gate_citation.mjs +167 -0
- package/tests/government_protected_surfaces.mjs +14 -0
- package/tests/grade_model_provider.mjs +217 -0
- package/tests/grader.mjs +48 -126
- package/tests/grader_unbacked_score.mjs +1 -1
- package/tests/regrade_actual_minutes.mjs +18 -17
- package/tests/render_check.mjs +1 -1
- package/tests/skill_grade_audit.mjs +3 -3
- package/tests/skill_grader_health.mjs +2 -2
- package/scripts/gds/audit-ships.js +0 -319
- package/tests/audit_ships_git_argv.mjs +0 -173
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
const path = require('node:path');
|
|
13
13
|
const fs = require('node:fs');
|
|
14
14
|
const { DEFAULT_GRADER_MODEL, FALLTHROUGH_CONFIDENCE, LOW_CONFIDENCE, THRESHOLD, VERSION, permissionPathCheck, routeRankCheck } = require('./grader-config.js');
|
|
15
|
-
const { TRIVIAL_MAX_FILES, TRIVIAL_MAX_LINES, clamp,
|
|
15
|
+
const { TRIVIAL_MAX_FILES, TRIVIAL_MAX_LINES, clamp, r1 } = require('./grader-model.js');
|
|
16
16
|
const { extractJson } = require('./grader-prompt.js');
|
|
17
17
|
const { issueClaimsRuntimeDead } = require('./dead-feature-claim.js');
|
|
18
18
|
|
|
@@ -612,173 +612,6 @@ function serverSideGuards(gradeResult, { committedFiles, builderRank, diffStats,
|
|
|
612
612
|
};
|
|
613
613
|
}
|
|
614
614
|
|
|
615
|
-
// ---------- v3: multi-shot aggregation ----------
|
|
616
|
-
|
|
617
|
-
// Collapses N per-shot grade-results into the final v3 grade-result.
|
|
618
|
-
// Pass requires unanimous PASS on all shots; anything else (any FAIL, any
|
|
619
|
-
// error, any disagreement) → not passed. Issues are deduped across shots
|
|
620
|
-
// by (file, summary). Displayed rubric_scores use the median across shots
|
|
621
|
-
// so the trend dashboard stays one-number-per-task.
|
|
622
|
-
function aggregateShots(shotResults, meta = {}) {
|
|
623
|
-
const N = shotResults.length;
|
|
624
|
-
const verdicts = shotResults.map((s) => ({
|
|
625
|
-
passed: s.passed,
|
|
626
|
-
ff: s.rubric_scores?.functional_fidelity ?? null,
|
|
627
|
-
cq: s.rubric_scores?.code_quality ?? null,
|
|
628
|
-
economy: s.rubric_scores?.economy ?? null,
|
|
629
|
-
cq_effective: s.rubric_scores?.code_quality_effective ?? null,
|
|
630
|
-
confidence: s.signals?.confidence ?? null,
|
|
631
|
-
model: s.signals?.model ?? null,
|
|
632
|
-
run_ms: s.signals?.run_ms ?? null,
|
|
633
|
-
shot_index: s.signals?.shot_index ?? null,
|
|
634
|
-
notes_md: s.notes_md,
|
|
635
|
-
issues: s.issues || [],
|
|
636
|
-
error:
|
|
637
|
-
s.signals?.parse_error
|
|
638
|
-
? 'parse_error'
|
|
639
|
-
: s.signals?.schema_error
|
|
640
|
-
? 'schema_error'
|
|
641
|
-
: s.signals?.subprocess_error || null,
|
|
642
|
-
}));
|
|
643
|
-
|
|
644
|
-
const ff = median(verdicts.map((v) => v.ff));
|
|
645
|
-
const cq = median(verdicts.map((v) => v.cq));
|
|
646
|
-
const economy = median(verdicts.map((v) => v.economy));
|
|
647
|
-
const cqEffective = median(verdicts.map((v) => v.cq_effective));
|
|
648
|
-
// NB (task 1003318): this v3 multi-shot path still collapses a null axis to
|
|
649
|
-
// 0 — it serves the trivial-diff single shot, which cannot overflow a window;
|
|
650
|
-
// the panel path's lens-outage handling lives in validateAndScore.
|
|
651
|
-
const blended = ff != null && cq != null ? r1((ff + cq) / 2) : 0;
|
|
652
|
-
// Top-level aggregated confidence = median of per-shot confidences. Without
|
|
653
|
-
// this, ship.js forwards a null top-level confidence to /tasks/:id/grade,
|
|
654
|
-
// and the server's validateAndScore (re-running on the POST body) hits the
|
|
655
|
-
// Number(null)=0 trap → FAIL even though every shot independently passed.
|
|
656
|
-
// Caught on the MAS Phase 1 self-ship (#395).
|
|
657
|
-
//
|
|
658
|
-
// Computed UNROUNDED (no r1()) — confidence sits near a 0.4 gate cap so
|
|
659
|
-
// rounding to one decimal could nudge a borderline value across the cap
|
|
660
|
-
// in either direction. Per-shot confidences are already 0-1 floats.
|
|
661
|
-
const confVals = verdicts
|
|
662
|
-
.map((v) => v.confidence)
|
|
663
|
-
.filter((n) => Number.isFinite(n))
|
|
664
|
-
.sort((a, b) => a - b);
|
|
665
|
-
const confidenceMedian = confVals.length === 0
|
|
666
|
-
? null
|
|
667
|
-
: confVals.length % 2
|
|
668
|
-
? confVals[(confVals.length - 1) / 2]
|
|
669
|
-
: (confVals[confVals.length / 2 - 1] + confVals[confVals.length / 2]) / 2;
|
|
670
|
-
|
|
671
|
-
// Dedupe issues across shots by (file, summary). Keep first-seen severity.
|
|
672
|
-
const seen = new Set();
|
|
673
|
-
const issues = [];
|
|
674
|
-
for (const v of verdicts) {
|
|
675
|
-
for (const it of v.issues) {
|
|
676
|
-
if (!it || typeof it !== 'object') continue;
|
|
677
|
-
const key = `${it.file ?? ''}::${(it.summary ?? '').trim()}`;
|
|
678
|
-
if (seen.has(key)) continue;
|
|
679
|
-
seen.add(key);
|
|
680
|
-
issues.push(it);
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
const passCount = verdicts.filter((v) => v.passed === true).length;
|
|
685
|
-
const failCount = verdicts.filter((v) => v.passed === false && !v.error).length;
|
|
686
|
-
const errorCount = verdicts.filter((v) => v.error).length;
|
|
687
|
-
const allPassed = N > 0 && passCount === N;
|
|
688
|
-
|
|
689
|
-
let verdict;
|
|
690
|
-
let passed;
|
|
691
|
-
let aggregationKind;
|
|
692
|
-
if (errorCount > 0) {
|
|
693
|
-
passed = false;
|
|
694
|
-
verdict = `FAIL — ${errorCount}/${N} shot(s) errored`;
|
|
695
|
-
aggregationKind = 'errored';
|
|
696
|
-
} else if (allPassed) {
|
|
697
|
-
passed = true;
|
|
698
|
-
verdict = `PASS — ${N}/${N} shots unanimous`;
|
|
699
|
-
aggregationKind = 'unanimous_pass';
|
|
700
|
-
} else if (passCount === 0) {
|
|
701
|
-
passed = false;
|
|
702
|
-
verdict = `FAIL — ${failCount}/${N} shots`;
|
|
703
|
-
aggregationKind = 'unanimous_fail';
|
|
704
|
-
} else {
|
|
705
|
-
passed = false;
|
|
706
|
-
verdict = `FAIL — ${passCount} pass, ${failCount} fail (no unanimous pass)`;
|
|
707
|
-
aggregationKind = 'disagreement';
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
const lines = [];
|
|
711
|
-
lines.push(`Subagent grader ${VERSION} (multi-shot, ${N}× shots) — ${verdict}`);
|
|
712
|
-
lines.push(` functional_fidelity (median): ${ff ?? '?'}/10`);
|
|
713
|
-
lines.push(` code_quality (median): ${cq ?? '?'}/10`);
|
|
714
|
-
if (economy != null) {
|
|
715
|
-
lines.push(` economy (advisory, median): ${economy}/10`);
|
|
716
|
-
}
|
|
717
|
-
lines.push(` blended (median): ${blended}/10 (threshold ${THRESHOLD}, both axes must clear on EVERY shot)`);
|
|
718
|
-
lines.push('');
|
|
719
|
-
lines.push('Per-shot:');
|
|
720
|
-
for (let i = 0; i < verdicts.length; i++) {
|
|
721
|
-
const v = verdicts[i];
|
|
722
|
-
const status = v.error ? 'ERROR' : v.passed === true ? 'PASS' : 'FAIL';
|
|
723
|
-
const ffStr = v.ff != null ? `${v.ff}` : '?';
|
|
724
|
-
const cqStr = v.cq != null ? `${v.cq}` : '?';
|
|
725
|
-
const confStr = v.confidence != null ? ` conf=${v.confidence}` : '';
|
|
726
|
-
const errStr = v.error ? ` (${v.error})` : '';
|
|
727
|
-
lines.push(` shot[${i}] ${status} ff=${ffStr} cq=${cqStr}${confStr} ${v.model ?? '?'}${errStr}`);
|
|
728
|
-
}
|
|
729
|
-
if (issues.length > 0) {
|
|
730
|
-
lines.push('');
|
|
731
|
-
lines.push(`Issues (${issues.length} unique across shots):`);
|
|
732
|
-
for (const it of issues.slice(0, 20)) {
|
|
733
|
-
const sev = String(it.severity || '?').padEnd(7);
|
|
734
|
-
const file = String(it.file || '?');
|
|
735
|
-
const summary = String(it.summary || '').replace(/\s+/g, ' ').trim();
|
|
736
|
-
lines.push(` [${sev}] ${file} — ${summary}`);
|
|
737
|
-
}
|
|
738
|
-
if (issues.length > 20) {
|
|
739
|
-
lines.push(` ... ${issues.length - 20} more (full set in signals.shots[].issues)`);
|
|
740
|
-
}
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
// #486 (ADR 0028 path 1): sum each shot's own subprocess cost (envelope
|
|
744
|
-
// total_cost_usd) so the trivial-diff path reports panel_cost_usd the same way
|
|
745
|
-
// the full panel does. The single-shot trivial path is the common case here.
|
|
746
|
-
const panelCostUsd = shotResults.reduce((s, r) => s + (Number(r.cost_usd) || 0), 0);
|
|
747
|
-
|
|
748
|
-
return {
|
|
749
|
-
score: blended,
|
|
750
|
-
passed,
|
|
751
|
-
threshold: THRESHOLD,
|
|
752
|
-
grader_kind: 'subagent-multishot',
|
|
753
|
-
grader_version: VERSION,
|
|
754
|
-
panel_cost_usd: panelCostUsd, // #486: panel's own API spend (USD)
|
|
755
|
-
rubric_scores: {
|
|
756
|
-
functional_fidelity: ff,
|
|
757
|
-
code_quality: cq,
|
|
758
|
-
economy,
|
|
759
|
-
code_quality_effective: cqEffective,
|
|
760
|
-
},
|
|
761
|
-
signals: {
|
|
762
|
-
grader_kind: 'subagent-multishot',
|
|
763
|
-
grader_version: VERSION,
|
|
764
|
-
threshold: THRESHOLD,
|
|
765
|
-
// Top-level confidence = median across shots. ship.js forwards this to
|
|
766
|
-
// POST /tasks/:id/grade so the server-side re-validation has a real
|
|
767
|
-
// number rather than null (see Number(null)=0 fix in validateAndScore).
|
|
768
|
-
confidence: confidenceMedian,
|
|
769
|
-
shots: verdicts,
|
|
770
|
-
shot_count: N,
|
|
771
|
-
pass_count: passCount,
|
|
772
|
-
fail_count: failCount,
|
|
773
|
-
error_count: errorCount,
|
|
774
|
-
shot_aggregation: aggregationKind,
|
|
775
|
-
...meta,
|
|
776
|
-
},
|
|
777
|
-
notes_md: lines.join('\n'),
|
|
778
|
-
issues,
|
|
779
|
-
};
|
|
780
|
-
}
|
|
781
|
-
|
|
782
615
|
// ---------- v3: subagent subprocess invoker ----------
|
|
783
616
|
|
|
784
617
|
// Unchanged from v2 — spawns `claude -p --print --model <model> --output-
|
|
@@ -786,7 +619,6 @@ function aggregateShots(shotResults, meta = {}) {
|
|
|
786
619
|
// via opts.model (set by gradeSubagent below).
|
|
787
620
|
|
|
788
621
|
module.exports = {
|
|
789
|
-
aggregateShots,
|
|
790
622
|
changeMagnitude,
|
|
791
623
|
hasPanelEvidence,
|
|
792
624
|
isSubstantiveIssue,
|
|
@@ -11,10 +11,14 @@
|
|
|
11
11
|
const path = require('node:path');
|
|
12
12
|
const fs = require('node:fs');
|
|
13
13
|
const { spawn } = require('node:child_process');
|
|
14
|
-
const { DEFAULT_GRADER_MODEL,
|
|
15
|
-
|
|
14
|
+
const { DEFAULT_GRADER_MODEL, api } = require('./grader-config.js');
|
|
15
|
+
// task 1002840: the ModelProvider port (ADR 0089 §3) — runSubagent below routes
|
|
16
|
+
// through it; the Claude spawn is its default provider.
|
|
17
|
+
const modelProvider = require('./model-provider.js');
|
|
16
18
|
|
|
17
|
-
|
|
19
|
+
// The `claude -p` spawn — the anthropic-cli ModelProvider. Body unchanged by
|
|
20
|
+
// task 1002840; only its name moved, so runSubagent could become the port.
|
|
21
|
+
function runClaudeCli({ prompt, opts = {} }) {
|
|
18
22
|
return new Promise((resolve, reject) => {
|
|
19
23
|
const timeoutMs = opts.timeoutMs ?? 240_000;
|
|
20
24
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -120,6 +124,17 @@ function runSubagent({ prompt, opts = {} }) {
|
|
|
120
124
|
child.stdin.end();
|
|
121
125
|
});
|
|
122
126
|
}
|
|
127
|
+
modelProvider.registerModelProvider({ name: modelProvider.DEFAULT_PROVIDER, kind: 'anthropic-cli', run: runClaudeCli });
|
|
128
|
+
|
|
129
|
+
// runSubagent — the ModelProvider port every platform LLM call goes through
|
|
130
|
+
// (the panel workers, the no-artifact shots, architect-audit, the agents module
|
|
131
|
+
// via the grade port). A model with no registered provider prefix — every id in
|
|
132
|
+
// use today — runs on the Claude spawn exactly as before; `<provider>:<model>`
|
|
133
|
+
// runs on that provider, which answers in the same envelope + error codes.
|
|
134
|
+
function runSubagent({ prompt, opts = {} }) {
|
|
135
|
+
const ref = modelProvider.parseModelRef(opts.model ?? DEFAULT_GRADER_MODEL);
|
|
136
|
+
return modelProvider.resolveModelProvider(ref.provider).run({ prompt, opts: { ...opts, model: ref.model } });
|
|
137
|
+
}
|
|
123
138
|
|
|
124
139
|
// ---------- task 1002660: shared context bundle ----------
|
|
125
140
|
//
|
|
@@ -261,54 +276,6 @@ function extractAssistantResult(rawStdout) {
|
|
|
261
276
|
}
|
|
262
277
|
}
|
|
263
278
|
|
|
264
|
-
// Run one shot end-to-end: build the prompt (caller did this), spawn the
|
|
265
|
-
// subagent at the requested model, parse + validate. Returns a per-shot
|
|
266
|
-
// grade-result; subprocess failures become passed=false per-shot results so
|
|
267
|
-
// aggregateShots can still produce a final verdict.
|
|
268
|
-
async function runOneShot({ prompt, model, shotIndex, runOpts }) {
|
|
269
|
-
try {
|
|
270
|
-
const { stdout, run_ms, exit_code, cost_usd } = await runSubagent({
|
|
271
|
-
prompt,
|
|
272
|
-
opts: { ...runOpts, model },
|
|
273
|
-
});
|
|
274
|
-
const assistantResult = extractAssistantResult(stdout);
|
|
275
|
-
const scored = validateAndScore(assistantResult, {
|
|
276
|
-
run_ms,
|
|
277
|
-
exit_code,
|
|
278
|
-
raw_chars: stdout.length,
|
|
279
|
-
model,
|
|
280
|
-
shot_index: shotIndex,
|
|
281
|
-
});
|
|
282
|
-
// #486 (ADR 0028 path 1): carry this shot's own subprocess cost so the
|
|
283
|
-
// trivial-diff path (gradeSubagentV3MultiShot → aggregateShots) can sum it
|
|
284
|
-
// into panel_cost_usd just like the full panel does.
|
|
285
|
-
scored.cost_usd = cost_usd ?? null;
|
|
286
|
-
return scored;
|
|
287
|
-
} catch (err) {
|
|
288
|
-
return {
|
|
289
|
-
score: 0,
|
|
290
|
-
passed: false,
|
|
291
|
-
threshold: THRESHOLD,
|
|
292
|
-
grader_kind: 'subagent',
|
|
293
|
-
grader_version: VERSION,
|
|
294
|
-
rubric_scores: {},
|
|
295
|
-
signals: {
|
|
296
|
-
grader_kind: 'subagent',
|
|
297
|
-
grader_version: VERSION,
|
|
298
|
-
model,
|
|
299
|
-
threshold: THRESHOLD,
|
|
300
|
-
shot_index: shotIndex,
|
|
301
|
-
run_ms: err.run_ms ?? null,
|
|
302
|
-
exit_code: err.exit_code ?? null,
|
|
303
|
-
subprocess_error: err.code || 'UNKNOWN',
|
|
304
|
-
stderr_tail: (err.stderr || '').slice(-500),
|
|
305
|
-
},
|
|
306
|
-
notes_md: `Shot ${shotIndex}: subprocess failed (${err.code || 'UNKNOWN'}: ${err.message}).`,
|
|
307
|
-
issues: [],
|
|
308
|
-
};
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
|
|
312
279
|
// ---------- v4: panel aggregation (MAS Phase 2.0) ----------
|
|
313
280
|
|
|
314
281
|
// Collapses N per-worker WorkerResults into the final v4 grade-result.
|
|
@@ -342,7 +309,6 @@ module.exports = {
|
|
|
342
309
|
diffAddedLinesByFile,
|
|
343
310
|
extractAssistantResult,
|
|
344
311
|
extractCostUsd,
|
|
345
|
-
runOneShot,
|
|
346
312
|
runSubagent,
|
|
347
313
|
runSubagentCached,
|
|
348
314
|
};
|
|
@@ -75,23 +75,28 @@
|
|
|
75
75
|
// module (which receives it via dependency injection to avoid a circular
|
|
76
76
|
// require).
|
|
77
77
|
//
|
|
78
|
-
// ##
|
|
78
|
+
// ## Retired (task 1002842)
|
|
79
79
|
//
|
|
80
|
-
//
|
|
81
|
-
// scripts/gds/audit-ships.js
|
|
80
|
+
// The v1 deterministic process-audit grader (gradeDeterministic and its nine
|
|
81
|
+
// per-criterion scorers) and its one caller, scripts/gds/audit-ships.js, are
|
|
82
|
+
// deleted: the live panel grades every ship at ship time, so a retroactive
|
|
83
|
+
// heuristic audit has nothing left to cover. aggregateShots / runOneShot moved
|
|
84
|
+
// to grader-no-artifact.js, the species that depends on them; they are still
|
|
85
|
+
// re-exported here, and the trivial-diff fast path below still runs one shot.
|
|
82
86
|
|
|
83
87
|
const path = require('node:path');
|
|
84
88
|
const fs = require('node:fs');
|
|
85
89
|
const { spawn } = require('node:child_process');
|
|
86
90
|
|
|
87
|
-
const { ASSUMED_BUILDER_MODEL, DEFAULT_GRADER_MODEL,
|
|
91
|
+
const { ASSUMED_BUILDER_MODEL, DEFAULT_GRADER_MODEL, FALLTHROUGH_CONFIDENCE, RUBRIC, THRESHOLD, VERSION, generatedProvenance, permissionPathPrePass, routeRankPrePass, workers } = require('./grader-config.js');
|
|
88
92
|
const { clamp, isTrivialDiff, median, r1, selectGraderModel, workerModelFor } = require('./grader-model.js');
|
|
89
93
|
const { buildPrompt, extractJson } = require('./grader-prompt.js');
|
|
90
94
|
// task 1004011: the second grade species — a grade over the CLAIM on an
|
|
91
95
|
// --allow-empty ship, where there is no diff for the panel above to read.
|
|
92
|
-
|
|
93
|
-
const {
|
|
94
|
-
const {
|
|
96
|
+
// task 1002842: the generalist-shot plumbing now lives there too.
|
|
97
|
+
const { GRADER_KIND: NO_ARTIFACT_GRADER_KIND, aggregateShots, buildNoArtifactPrompt, gradeNoArtifact, runOneShot } = require('./grader-no-artifact.js');
|
|
98
|
+
const { changeMagnitude, hasPanelEvidence, isSubstantiveIssue, serverSideGuards, validateAndScore } = require('./grader-score.js');
|
|
99
|
+
const { buildContextBundle, diffAddedLinesByFile, extractAssistantResult, extractCostUsd, runSubagent, runSubagentCached } = require('./grader-subagent.js');
|
|
95
100
|
// task 1002903: the severity gate's unbacked low scores, named once for the record.
|
|
96
101
|
const { unbackedLowScores } = require('./grader-workers/worker-verdict.js');
|
|
97
102
|
// task 1002839: the central engine the code panel below registers into (ADR 0089 §1).
|
|
@@ -478,8 +483,8 @@ function aggregatePanel(workerResults, meta = {}) {
|
|
|
478
483
|
grader_version: VERSION,
|
|
479
484
|
// #486: the panel's own API spend (USD), for cost_log (source=grader).
|
|
480
485
|
panel_cost_usd: panelCostUsd,
|
|
481
|
-
// rubric_scores keep the v3 keys so the trend dashboard
|
|
482
|
-
//
|
|
486
|
+
// rubric_scores keep the v3 keys so the trend dashboard and the server's
|
|
487
|
+
// validateAndScore re-validation continue to read them. Phase 2.1 added
|
|
483
488
|
// security_score; Phase 2.2 adds efficiency_score. Both additive —
|
|
484
489
|
// consumers that don't know about them just ignore the fields.
|
|
485
490
|
rubric_scores: {
|
|
@@ -651,12 +656,17 @@ async function gradeSubagent({ task, valueSummary, diff, changedFiles, diffStats
|
|
|
651
656
|
});
|
|
652
657
|
}
|
|
653
658
|
|
|
654
|
-
// ----------
|
|
659
|
+
// ---------- the trivial-diff fast path (#513) ----------
|
|
655
660
|
//
|
|
656
|
-
//
|
|
657
|
-
// (
|
|
658
|
-
//
|
|
659
|
-
|
|
661
|
+
// A small copy/docs-only diff gets ONE generalist shot instead of the four-worker
|
|
662
|
+
// panel — "lighter, not skipped" (ADR 0089 §7 keeps it). The name is historical:
|
|
663
|
+
// the v3 multi-shot GENERATION is retired (task 1002842 removed its 3-shot default
|
|
664
|
+
// and rubric block), and this is what is left of it — the working fast path.
|
|
665
|
+
// Moving it onto a v4 panel needs a reviewer type that reports both
|
|
666
|
+
// functional_fidelity and code_quality, which no v4 worker does; that is a cost
|
|
667
|
+
// and behaviour decision, not a cleanup. `runShot` is a test seam defaulted to
|
|
668
|
+
// the real spawner (the same pattern as gradeNoArtifact); production passes none.
|
|
669
|
+
async function gradeSubagentV3MultiShot({ task, valueSummary, diff, changedFiles, diffStats, shots = 1, builderModel, runOpts = {}, runShot = runOneShot }) {
|
|
660
670
|
const prompt = buildPrompt({ task, valueSummary, diff, changedFiles, diffStats, generatedProvenance: await generatedProvenance.verifyGeneratedProvenanceAsync({ changedFiles, cwd: runOpts.cwd }) }); // task 1002795
|
|
661
671
|
const graderModel = selectGraderModel({ builderModel });
|
|
662
672
|
const N = Math.max(1, Math.trunc(shots));
|
|
@@ -664,7 +674,7 @@ async function gradeSubagentV3MultiShot({ task, valueSummary, diff, changedFiles
|
|
|
664
674
|
const startedAt = Date.now();
|
|
665
675
|
const shotPromises = [];
|
|
666
676
|
for (let i = 0; i < N; i++) {
|
|
667
|
-
shotPromises.push(
|
|
677
|
+
shotPromises.push(runShot({ prompt, model: graderModel, shotIndex: i, runOpts }));
|
|
668
678
|
}
|
|
669
679
|
const shotResults = await Promise.all(shotPromises);
|
|
670
680
|
const totalMs = Date.now() - startedAt;
|
|
@@ -676,189 +686,6 @@ async function gradeSubagentV3MultiShot({ task, valueSummary, diff, changedFiles
|
|
|
676
686
|
});
|
|
677
687
|
}
|
|
678
688
|
|
|
679
|
-
// =========================================================================
|
|
680
|
-
// Legacy v1 (deterministic) — kept for scripts/gds/audit-ships.js only.
|
|
681
|
-
// =========================================================================
|
|
682
|
-
|
|
683
|
-
function scoreEstActualBand(actualMinutes, estMinutes) {
|
|
684
|
-
if (!Number.isFinite(actualMinutes) || actualMinutes <= 0) return 5;
|
|
685
|
-
if (!Number.isFinite(estMinutes) || estMinutes <= 0) return 6;
|
|
686
|
-
const ratio = actualMinutes / estMinutes;
|
|
687
|
-
if (ratio >= 0.5 && ratio <= 2.0) return 10;
|
|
688
|
-
if (ratio < 0.5) {
|
|
689
|
-
if (ratio <= 0.1) return 0;
|
|
690
|
-
return r1(10 * ((ratio - 0.1) / 0.4));
|
|
691
|
-
}
|
|
692
|
-
if (ratio > 2.0) {
|
|
693
|
-
if (ratio >= 10) return 0;
|
|
694
|
-
return r1(10 * (1 - (ratio - 2.0) / 8.0));
|
|
695
|
-
}
|
|
696
|
-
return 5;
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
function scoreValueSummary(valueSummary) {
|
|
700
|
-
const s = (valueSummary || '').trim();
|
|
701
|
-
if (s.length === 0) return 0;
|
|
702
|
-
if (s.length < 40) return 5;
|
|
703
|
-
if (s.length < 120) return 8;
|
|
704
|
-
return /\b(add|fix|update|remove|refactor|wire|land|migrate|enable|disable|introduce|deprecate|ship|deploy|prevent|capture|gate|allocate|carry|expose|surface)\b/i.test(s)
|
|
705
|
-
? 10
|
|
706
|
-
: 8;
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
function scoreDiffHygiene(diffStats) {
|
|
710
|
-
if (!diffStats) return 5;
|
|
711
|
-
const files = Number(diffStats.files_changed) || 0;
|
|
712
|
-
const added = Number(diffStats.lines_added) || 0;
|
|
713
|
-
const removed = Number(diffStats.lines_removed) || 0;
|
|
714
|
-
if (files === 0 || added + removed === 0) return 0;
|
|
715
|
-
if (files > 50) return 5;
|
|
716
|
-
return 10;
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
function scoreScannerSmoke(verification) {
|
|
720
|
-
if (!verification) return 5;
|
|
721
|
-
return verification.scanner_clean && verification.smoke_clean ? 10 : 5;
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
function scoreScopingSignal(scopingConfidence) {
|
|
725
|
-
if (scopingConfidence == null) return 6;
|
|
726
|
-
const c = Number(scopingConfidence);
|
|
727
|
-
if (c <= 0) return 2;
|
|
728
|
-
if (c === 1) return 4;
|
|
729
|
-
if (c === 2) return 6;
|
|
730
|
-
if (c === 3) return 7;
|
|
731
|
-
if (c === 4) return 8;
|
|
732
|
-
return 10;
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
function scoreLocNetNegative(diffStats) {
|
|
736
|
-
if (!diffStats) return 5;
|
|
737
|
-
const added = Number(diffStats.lines_added) || 0;
|
|
738
|
-
const removed = Number(diffStats.lines_removed) || 0;
|
|
739
|
-
if (removed > added) return 10;
|
|
740
|
-
if (removed === added) return 5;
|
|
741
|
-
return 0;
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
function scoreLearningRow(learningRow) {
|
|
745
|
-
if (!learningRow) return 0;
|
|
746
|
-
const body = (learningRow.body_md || '').trim();
|
|
747
|
-
if (body.length > 100) return 10;
|
|
748
|
-
if (body.length > 0) return 5;
|
|
749
|
-
return 0;
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
function scoreArtRubricAvg(value) {
|
|
753
|
-
if (!Number.isFinite(value)) return null;
|
|
754
|
-
return clamp(r1(value), 0, 10);
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
function blendWeightsForKind(kind) {
|
|
758
|
-
const blended = {};
|
|
759
|
-
for (const [name, def] of Object.entries(V1.weights.common)) {
|
|
760
|
-
blended[name] = def.weight;
|
|
761
|
-
}
|
|
762
|
-
const kindBonus = V1.weights.kind_specific[kind];
|
|
763
|
-
if (kindBonus) {
|
|
764
|
-
for (const [name, def] of Object.entries(kindBonus)) {
|
|
765
|
-
if (name === 'loc_net_negative' && 'diff_hygiene' in blended) delete blended.diff_hygiene;
|
|
766
|
-
if (name === 'learning_row' && 'diff_hygiene' in blended) delete blended.diff_hygiene;
|
|
767
|
-
blended[name] = def.weight;
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
const total = Object.values(blended).reduce((a, b) => a + b, 0);
|
|
771
|
-
for (const k of Object.keys(blended)) blended[k] = blended[k] / total;
|
|
772
|
-
return blended;
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
// Legacy v1 entry. Renamed from `grade()` in V3.R18 (#209). Now ONLY
|
|
776
|
-
// called from scripts/gds/audit-ships.js (retroactive #204 audit). The
|
|
777
|
-
// runtime ship path uses gradeSubagent (v3 multi-shot).
|
|
778
|
-
function gradeDeterministic({ task, verification, actualMinutes, valueSummary, learningRow }) {
|
|
779
|
-
const kind = task?.kind || 'unclassified';
|
|
780
|
-
const weights = blendWeightsForKind(kind);
|
|
781
|
-
const rubricScores = {};
|
|
782
|
-
const signals = {
|
|
783
|
-
kind,
|
|
784
|
-
est_minutes: task?.est_minutes ?? null,
|
|
785
|
-
actual_minutes: actualMinutes ?? null,
|
|
786
|
-
scoping_confidence: task?.scoping_confidence ?? null,
|
|
787
|
-
scanner_clean: verification?.scanner_clean ?? null,
|
|
788
|
-
smoke_clean: verification?.smoke_clean ?? null,
|
|
789
|
-
diff_stats: verification?.diff_stats ?? null,
|
|
790
|
-
art_rubric_avg: verification?.art_rubric_avg ?? null,
|
|
791
|
-
value_summary_chars: (valueSummary || '').trim().length,
|
|
792
|
-
};
|
|
793
|
-
|
|
794
|
-
if ('est_actual_band' in weights) {
|
|
795
|
-
rubricScores.est_actual_band = scoreEstActualBand(actualMinutes, task?.est_minutes);
|
|
796
|
-
}
|
|
797
|
-
if ('value_summary' in weights) {
|
|
798
|
-
rubricScores.value_summary = scoreValueSummary(valueSummary);
|
|
799
|
-
}
|
|
800
|
-
if ('diff_hygiene' in weights) {
|
|
801
|
-
rubricScores.diff_hygiene = scoreDiffHygiene(verification?.diff_stats);
|
|
802
|
-
}
|
|
803
|
-
if ('scanner_smoke' in weights) {
|
|
804
|
-
rubricScores.scanner_smoke = scoreScannerSmoke(verification);
|
|
805
|
-
}
|
|
806
|
-
if ('scoping_signal' in weights) {
|
|
807
|
-
rubricScores.scoping_signal = scoreScopingSignal(task?.scoping_confidence);
|
|
808
|
-
}
|
|
809
|
-
if ('loc_net_negative' in weights) {
|
|
810
|
-
rubricScores.loc_net_negative = scoreLocNetNegative(verification?.diff_stats);
|
|
811
|
-
}
|
|
812
|
-
if ('learning_row' in weights) {
|
|
813
|
-
rubricScores.learning_row = scoreLearningRow(learningRow);
|
|
814
|
-
}
|
|
815
|
-
if ('art_rubric_avg' in weights) {
|
|
816
|
-
const s = scoreArtRubricAvg(verification?.art_rubric_avg);
|
|
817
|
-
if (s == null) {
|
|
818
|
-
const w = weights.art_rubric_avg;
|
|
819
|
-
delete weights.art_rubric_avg;
|
|
820
|
-
const remaining = Object.values(weights).reduce((a, b) => a + b, 0);
|
|
821
|
-
if (remaining > 0) {
|
|
822
|
-
for (const k of Object.keys(weights)) weights[k] = weights[k] / remaining;
|
|
823
|
-
}
|
|
824
|
-
} else {
|
|
825
|
-
rubricScores.art_rubric_avg = s;
|
|
826
|
-
}
|
|
827
|
-
}
|
|
828
|
-
|
|
829
|
-
let score = 0;
|
|
830
|
-
for (const [name, w] of Object.entries(weights)) {
|
|
831
|
-
if (rubricScores[name] != null) score += rubricScores[name] * w;
|
|
832
|
-
}
|
|
833
|
-
score = r1(clamp(score, 0, 10));
|
|
834
|
-
|
|
835
|
-
const threshold = V1.threshold;
|
|
836
|
-
const passed = score >= threshold;
|
|
837
|
-
|
|
838
|
-
const lines = [];
|
|
839
|
-
lines.push(`Auto-grader ${V1.version} (deprecated) — task_kind=${kind}.`);
|
|
840
|
-
lines.push(`Score ${score}/10 vs threshold ${threshold} → ${passed ? 'PASS' : 'FAIL'}.`);
|
|
841
|
-
lines.push('');
|
|
842
|
-
lines.push('Per-criterion:');
|
|
843
|
-
for (const [name, w] of Object.entries(weights)) {
|
|
844
|
-
const s = rubricScores[name];
|
|
845
|
-
if (s == null) continue;
|
|
846
|
-
const contrib = r1(s * w);
|
|
847
|
-
lines.push(` - ${name.padEnd(20)} ${String(s).padStart(4)}/10 × ${w.toFixed(2)} = ${contrib}`);
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
return {
|
|
851
|
-
score,
|
|
852
|
-
passed,
|
|
853
|
-
threshold,
|
|
854
|
-
grader_kind: 'auto',
|
|
855
|
-
grader_version: V1.version,
|
|
856
|
-
rubric_scores: rubricScores,
|
|
857
|
-
signals,
|
|
858
|
-
notes_md: lines.join('\n'),
|
|
859
|
-
};
|
|
860
|
-
}
|
|
861
|
-
|
|
862
689
|
module.exports = {
|
|
863
690
|
// v4 surface (active panel-based gate)
|
|
864
691
|
aggregatePanel,
|
|
@@ -888,25 +715,13 @@ module.exports = {
|
|
|
888
715
|
runSubagent,
|
|
889
716
|
runSubagentCached, // System 5 (task 1359): opt-in content-addressed cached path
|
|
890
717
|
runOneShot,
|
|
891
|
-
gradeSubagentV3MultiShot, //
|
|
718
|
+
gradeSubagentV3MultiShot, // the trivial-diff fast path (#513) — one generalist shot
|
|
892
719
|
extractJson,
|
|
893
720
|
extractAssistantResult,
|
|
894
721
|
extractCostUsd, // #486: parse total_cost_usd from the envelope
|
|
895
722
|
THRESHOLD,
|
|
896
723
|
VERSION,
|
|
897
724
|
DEFAULT_GRADER_MODEL,
|
|
898
|
-
DEFAULT_SHOTS,
|
|
899
725
|
ASSUMED_BUILDER_MODEL,
|
|
900
|
-
// legacy v1 (audit-ships.js only)
|
|
901
|
-
gradeDeterministic,
|
|
902
|
-
scoreEstActualBand,
|
|
903
|
-
scoreValueSummary,
|
|
904
|
-
scoreDiffHygiene,
|
|
905
|
-
scoreScannerSmoke,
|
|
906
|
-
scoreScopingSignal,
|
|
907
|
-
scoreLocNetNegative,
|
|
908
|
-
scoreLearningRow,
|
|
909
|
-
scoreArtRubricAvg,
|
|
910
|
-
blendWeightsForKind,
|
|
911
726
|
RUBRIC,
|
|
912
727
|
};
|