@bongos/core 1.19.332 → 1.19.334
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +173 -43
- package/docs/adr/0187-oauth-handshake-cookie-lifetime.md +84 -0
- package/docs/adr/README.md +1 -0
- package/docs/api/openapi.json +2 -2
- package/docs/api-reference.md +1 -1
- package/docs/copy-inventory.md +202 -202
- package/docs/copy-registry.json +384 -384
- package/modules/grading/grader-config.js +98 -0
- package/modules/grading/grader-model.js +150 -0
- package/modules/grading/grader-prompt.js +139 -0
- package/modules/grading/grader-score.js +706 -0
- package/modules/grading/grader-subagent.js +328 -0
- package/modules/grading/grader.js +5 -1287
- package/modules/hall-ui/public/builders.js +19 -1182
- package/modules/hall-ui/public/goals-page.js +11 -997
- package/modules/hall-ui/public/goals-render.js +1032 -0
- package/modules/hall-ui/public/goals.html +4 -0
- package/modules/hall-ui/public/hall-render.js +1211 -0
- package/modules/hall-ui/public/index.html +6 -0
- package/modules/lifecycle/routes/goal-route-authz.js +400 -0
- package/modules/lifecycle/routes/goals.js +1 -367
- package/modules/lifecycle/routes/task-route-helpers.js +219 -0
- package/modules/lifecycle/routes/task-write-routes.js +542 -0
- package/modules/lifecycle/routes/tasks.js +3 -696
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/scripts/gds/box-infra.js +280 -0
- package/scripts/gds/box.js +1 -245
- package/scripts/gds/fitness-checks-identity.js +320 -0
- package/scripts/gds/fitness-checks-vocabulary.js +230 -0
- package/scripts/gds/fitness-checks-write-validation.js +144 -0
- package/scripts/gds/fitness-lib.js +82 -0
- package/scripts/gds/fitness.js +55 -668
- package/scripts/gds/provision-config.js +204 -0
- package/scripts/gds/provision-net.js +221 -0
- package/scripts/gds/provision-repo.js +849 -0
- package/scripts/gds/provision-units.js +262 -0
- package/scripts/gds/provision.js +4 -1395
- package/src/bongos/auth-admission.js +264 -0
- package/src/bongos/auth-config.js +151 -0
- package/src/bongos/auth-discord.js +198 -0
- package/src/bongos/auth-github.js +336 -0
- package/src/bongos/auth-idp.js +330 -0
- package/src/bongos/auth.js +5 -1100
- package/src/bongos/routes/auth.js +54 -10
- package/src/module-api.js +5 -1
- package/tests/auth_admission_gate.mjs +3 -1
- package/tests/auth_web_recovery.mjs +63 -0
- package/tests/est_advisory.mjs +3 -1
- package/tests/goal_advisory.mjs +3 -2
- package/tests/goal_criterion_confirm_ui.mjs +6 -0
- package/tests/goal_subtitle.mjs +5 -2
- package/tests/goals_card_view.mjs +10 -3
- package/tests/grader_generated_provenance.mjs +3 -1
- package/tests/hall_palette.mjs +3 -1
- package/tests/helpers.mjs +58 -25
- package/tests/lib_sh_resolution.mjs +3 -1
- package/tests/membership_kind_lockstep.mjs +3 -1
- package/tests/migration_allocation.mjs +5 -2
- package/tests/patch_gate_ordering.mjs +3 -1
- package/tests/rank_tier_single_source.mjs +16 -3
- package/tests/reward_gate.mjs +3 -2
|
@@ -82,1294 +82,12 @@ const path = require('node:path');
|
|
|
82
82
|
const fs = require('node:fs');
|
|
83
83
|
const { spawn } = require('node:child_process');
|
|
84
84
|
|
|
85
|
-
const RUBRIC =
|
|
86
|
-
|
|
87
|
-
);
|
|
85
|
+
const { ASSUMED_BUILDER_MODEL, DEFAULT_GRADER_MODEL, DEFAULT_SHOTS, FALLTHROUGH_CONFIDENCE, RUBRIC, THRESHOLD, V1, V3, VERSION, api, generatedProvenance, permissionPathPrePass, routeRankPrePass, workers } = require('./grader-config.js');
|
|
86
|
+
const { clamp, isTrivialDiff, median, r1, selectGraderModel, workerModelFor } = require('./grader-model.js');
|
|
87
|
+
const { buildPrompt, extractJson } = require('./grader-prompt.js');
|
|
88
|
+
const { aggregateShots, changeMagnitude, hasPanelEvidence, isSubstantiveIssue, serverSideGuards, validateAndScore } = require('./grader-score.js');
|
|
89
|
+
const { buildContextBundle, diffAddedLinesByFile, extractAssistantResult, extractCostUsd, runOneShot, runSubagent, runSubagentCached } = require('./grader-subagent.js');
|
|
88
90
|
|
|
89
|
-
// V4 = the active panel-based gate. V3 / V1 are kept under their fields for
|
|
90
|
-
// the two paths that still read them: V3 = the trivial-diff multi-shot grader
|
|
91
|
-
// (ship.js #513), V1 = the audit-ships.js retroactive deterministic path. The
|
|
92
|
-
// v2 binding + its rubric block were dead (no reader) and removed in task 466.
|
|
93
|
-
const V4 = RUBRIC.v4;
|
|
94
|
-
const V3 = RUBRIC.v3;
|
|
95
|
-
const V1 = RUBRIC.v1;
|
|
96
|
-
const THRESHOLD = V4.threshold;
|
|
97
|
-
const VERSION = V4.version;
|
|
98
|
-
const LOW_CONFIDENCE = V4.low_confidence_threshold;
|
|
99
|
-
// DEFAULT_SHOTS is kept for the v3 multi-shot path (still exported for
|
|
100
|
-
// diagnostics); v4 doesn't use shots — it uses workers.
|
|
101
|
-
const DEFAULT_SHOTS = V3.shots || 3;
|
|
102
|
-
const ASSUMED_BUILDER_MODEL = V4.assumed_builder_model || 'claude-fable-5';
|
|
103
|
-
const DEFAULT_GRADER_MODEL = V4.default_model || 'claude-sonnet-5';
|
|
104
|
-
const FALLTHROUGH_CONFIDENCE = V4.fallthrough_confidence ?? 0.3;
|
|
105
|
-
// task 1002660: per-worker model tier overrides (rubric-configured). The
|
|
106
|
-
// Quality worker was 41% of panel spend with 75% nit findings — its cq/economy
|
|
107
|
-
// judgment doesn't need the full cross-model tier. Keys are worker kinds;
|
|
108
|
-
// values are model ids. workerModelFor guards the cross-model invariant.
|
|
109
|
-
const WORKER_MODEL_TIERS = V4.worker_model_tiers || {};
|
|
110
|
-
const workers = require('./grader-workers');
|
|
111
|
-
const generatedProvenance = require('./generated-provenance'); // task 1002795
|
|
112
|
-
// Carved into modules/grading/ (BV1.R73 / ADR 0091 §1+§4): the two deterministic
|
|
113
|
-
// trust classifiers are KERNEL (KERNEL_FILES, R70) — reached through the doorway,
|
|
114
|
-
// never a deep core require. The worker panel + extract-json are intra-module.
|
|
115
|
-
const api = require('../../src/module-api');
|
|
116
|
-
const routeRankCheck = api.routeRankCheck;
|
|
117
|
-
const permissionPathCheck = api.permissionPathCheck;
|
|
118
|
-
|
|
119
|
-
// Deterministic pre-pass: the sub-Metic permission-path gate (SEC #863). Runs
|
|
120
|
-
// ONLY when the change set touches a permission-sensitive path AND the builder
|
|
121
|
-
// rank resolves below Metic — otherwise null (zero cost / no finding). Like the
|
|
122
|
-
// route-rank pre-pass, the result is a pseudo-worker appended to the panel so
|
|
123
|
-
// aggregatePanel's any-fail gate hard-fails a sub-Metic ship of protected code
|
|
124
|
-
// with no gate-logic change. builderRank comes from the caller's live /me read
|
|
125
|
-
// (ship.js) or req.builder.rank (server); unknown rank fails OPEN here (the
|
|
126
|
-
// post-push main audit, which reads the real author rank from the DB, owns the
|
|
127
|
-
// unknown/dishonest case).
|
|
128
|
-
function permissionPathPrePass(changedFiles, builderRank) {
|
|
129
|
-
const result = permissionPathCheck.checkPermissionPaths({ changedFiles, builderRank });
|
|
130
|
-
if (!result.hardFail) return null;
|
|
131
|
-
return { findings: result.findings, workerResult: permissionPathCheck.toWorkerResult(result) };
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
// Deterministic pre-pass: the route-rank invariant (#475 / C7). Runs ONLY when
|
|
135
|
-
// a src/bongos/routes/ file is in the change set (zero cost otherwise). Returns
|
|
136
|
-
// the findings (fed to the Hacker worker as input) plus a pseudo-worker result
|
|
137
|
-
// appended to the panel so aggregatePanel's any-fail gate handles it directly.
|
|
138
|
-
// Shared verbatim by gradeSubagent (ship-time) and conductor-dispatch.js
|
|
139
|
-
// (mid-flight) so the route-parsing logic lives in exactly one place.
|
|
140
|
-
function routeRankPrePass(changedFiles) {
|
|
141
|
-
const touched = (changedFiles || []).some((f) => routeRankCheck.ROUTE_FILE_RE.test(f));
|
|
142
|
-
if (!touched) return null;
|
|
143
|
-
const result = routeRankCheck.checkRouteRanks();
|
|
144
|
-
return { findings: result.findings, workerResult: routeRankCheck.toWorkerResult(result) };
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
function r1(n) {
|
|
148
|
-
return Math.round(Number(n) * 10) / 10;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function clamp(n, lo, hi) {
|
|
152
|
-
return Math.max(lo, Math.min(hi, n));
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
function median(arr) {
|
|
156
|
-
const v = arr.filter((n) => Number.isFinite(n)).slice().sort((a, b) => a - b);
|
|
157
|
-
if (v.length === 0) return null;
|
|
158
|
-
const mid = Math.floor(v.length / 2);
|
|
159
|
-
return v.length % 2 ? r1(v[mid]) : r1((v[mid - 1] + v[mid]) / 2);
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// ---------- v4.1: trivial-diff fast-path (#513) ----------
|
|
163
|
-
//
|
|
164
|
-
// The #506 post-mortem: a +1/-1 welcome-copy tweak ran the full 4-worker
|
|
165
|
-
// adversarial Opus/Sonnet panel — disproportionate cost for a one-word change.
|
|
166
|
-
// isTrivialDiff classifies a change as "copy/docs only, small, and touching no
|
|
167
|
-
// logic path". ship.js routes those to a SINGLE generalist grader shot instead
|
|
168
|
-
// of the panel ("lighter, not skipped" — the 1-shot still scores ff+cq+economy
|
|
169
|
-
// +confidence so the server gate applies, and the deterministic scanner + smoke
|
|
170
|
-
// + route-rank checks run regardless). Everything else keeps the full panel.
|
|
171
|
-
//
|
|
172
|
-
// Deliberately conservative — defaults to "not trivial" (full panel) whenever
|
|
173
|
-
// in doubt (missing diffStats, zero files, any logic-path file, oversized diff).
|
|
174
|
-
const TRIVIAL_MAX_FILES = 3;
|
|
175
|
-
const TRIVIAL_MAX_LINES = 30;
|
|
176
|
-
// Copy/docs/front-end surfaces eligible for the lighter single-shot grade.
|
|
177
|
-
// (Both web surfaces moved under modules/<key>/public/: the status dashboard to
|
|
178
|
-
// modules/status-ui/public/ in BV1.R84 and the hall to modules/hall-ui/public/ in
|
|
179
|
-
// BV1.R85.)
|
|
180
|
-
const TRIVIAL_PATH_RE = /(?:^|\/)[^/]+\.md$|^docs\/|^public\/|^modules\/hall-ui\/public\/|^modules\/status-ui\/public\//;
|
|
181
|
-
// Logic / infra / test surfaces that force the full panel even on a tiny diff —
|
|
182
|
-
// these are where a subtle one-liner can do real damage, so they never qualify.
|
|
183
|
-
const LOGIC_PATH_RE = /^src\/|^scripts\/|^migrations\/|^server\.js$|^tests\/|^\.claude\//;
|
|
184
|
-
// Executable code AND config are NEVER trivial, even under public*/ (#513
|
|
185
|
-
// Hacker). A copy tweak and an XSS / client-authz regression are
|
|
186
|
-
// indistinguishable to a path+size heuristic, and the builders-hall + game
|
|
187
|
-
// front-ends carry auth flows and game logic — so any code extension forces the
|
|
188
|
-
// full adversarial panel. Config/data files (.json/.yaml/.env/…) are excluded
|
|
189
|
-
// too: a client-side config change is security-relevant and shouldn't get the
|
|
190
|
-
// lighter shot. Only genuinely static content (markdown, CSS/HTML, images,
|
|
191
|
-
// plain text) takes the single shot.
|
|
192
|
-
// HTML and SVG are executable in a browser (inline <script>, on* handlers,
|
|
193
|
-
// <svg onload>), so they are NOT "static" for grading purposes — a small
|
|
194
|
-
// edit could introduce client-side XSS that only the full panel's Hacker
|
|
195
|
-
// would catch. Excluded alongside code + config (#513 Hacker).
|
|
196
|
-
const NON_TRIVIAL_EXT_RE = /\.(?:js|mjs|cjs|jsx|ts|tsx|py|sh|bash|rb|go|sql|json|ya?ml|toml|ini|env|html?|svg)$|(?:^|\/)\.env(?:\.|$)/;
|
|
197
|
-
|
|
198
|
-
function isTrivialDiff({ changedFiles, diffStats } = {}) {
|
|
199
|
-
if (!diffStats || typeof diffStats !== 'object') return false;
|
|
200
|
-
const files = Array.isArray(changedFiles) ? changedFiles.filter(Boolean) : [];
|
|
201
|
-
if (files.length === 0) return false;
|
|
202
|
-
const fc = Number(diffStats.files_changed);
|
|
203
|
-
const lines = (Number(diffStats.lines_added) || 0) + (Number(diffStats.lines_removed) || 0);
|
|
204
|
-
// Cap on BOTH the shortstat count and the actual file-list length — a rename
|
|
205
|
-
// can disagree between the two, so take the stricter of the two (#513 nit).
|
|
206
|
-
if (!Number.isFinite(fc) || fc <= 0 || fc > TRIVIAL_MAX_FILES) return false;
|
|
207
|
-
if (files.length > TRIVIAL_MAX_FILES) return false;
|
|
208
|
-
if (lines <= 0 || lines > TRIVIAL_MAX_LINES) return false;
|
|
209
|
-
// Every changed file must be a copy/docs path, none in a logic path, and none
|
|
210
|
-
// an executable-code or config file (those always get the full panel).
|
|
211
|
-
return files.every((f) =>
|
|
212
|
-
TRIVIAL_PATH_RE.test(f) && !LOGIC_PATH_RE.test(f) && !NON_TRIVIAL_EXT_RE.test(f));
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
// ---------- v3: cross-family model selection (ADR 0024) ----------
|
|
216
|
-
|
|
217
|
-
// Returns the grader model given the builder's model — a DIFFERENT model from
|
|
218
|
-
// the builder is the cross-model adversarial guarantee (ADR 0024). With no
|
|
219
|
-
// builderModel the caller assumes the owner-top-model builder (CLAUDE.md §2),
|
|
220
|
-
// resolving to the default grader — the path selectGraderModel({}) takes.
|
|
221
|
-
//
|
|
222
|
-
// task 1002659 (Claude 5 refresh): top-tier builders (fable/mythos, opus)
|
|
223
|
-
// grade on the current mid tier; sonnet/haiku builders grade on the tier
|
|
224
|
-
// above. #117's throw-on-unknown is reversed: the throw fired INSIDE the
|
|
225
|
-
// grade step, stranding any ship whose builder id lacked an opus/sonnet/haiku
|
|
226
|
-
// substring (claude-fable-5 was exactly that) — a worse failure than the
|
|
227
|
-
// misconfiguration it guarded. Unknown ids now warn loudly and grade on
|
|
228
|
-
// DEFAULT_GRADER_MODEL; the id still reaches signals.builder_model_assumed,
|
|
229
|
-
// so /grader-health can spot the drift.
|
|
230
|
-
function selectGraderModel({ builderModel } = {}) {
|
|
231
|
-
const bm = (builderModel || ASSUMED_BUILDER_MODEL).toLowerCase();
|
|
232
|
-
if (bm.includes('fable') || bm.includes('mythos')) return 'claude-sonnet-5';
|
|
233
|
-
if (bm.includes('opus')) return 'claude-sonnet-5';
|
|
234
|
-
if (bm.includes('sonnet')) return 'claude-opus-5';
|
|
235
|
-
if (bm.includes('haiku')) return 'claude-opus-5';
|
|
236
|
-
console.error(
|
|
237
|
-
`selectGraderModel: unrecognized builder model ${JSON.stringify(builderModel)} — ` +
|
|
238
|
-
`no 'fable'/'mythos'/'opus'/'sonnet'/'haiku' token; grading on the default ` +
|
|
239
|
-
`${DEFAULT_GRADER_MODEL} (cross-model unverifiable — task 1002659, was a #117 throw)`
|
|
240
|
-
);
|
|
241
|
-
return DEFAULT_GRADER_MODEL;
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// task 1002660: resolve one worker's model — the rubric tier override when one
|
|
245
|
-
// exists, else the panel's cross-model pick. The override is refused (falls
|
|
246
|
-
// back to graderModel) when it lands on the BUILDER's own family: tiering is
|
|
247
|
-
// a cost lever, never a hole in the ADR 0024 cross-model guarantee.
|
|
248
|
-
const MODEL_FAMILY_TOKENS = ['fable', 'mythos', 'opus', 'sonnet', 'haiku'];
|
|
249
|
-
function workerModelFor(kind, graderModel, builderModel) {
|
|
250
|
-
const tier = WORKER_MODEL_TIERS[kind];
|
|
251
|
-
if (!tier) return graderModel;
|
|
252
|
-
const bm = String(builderModel || ASSUMED_BUILDER_MODEL).toLowerCase();
|
|
253
|
-
const tierFamily = MODEL_FAMILY_TOKENS.find((t) => String(tier).toLowerCase().includes(t));
|
|
254
|
-
if (tierFamily && bm.includes(tierFamily)) return graderModel;
|
|
255
|
-
return tier;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
// ---------- v3: prompt builder (default-fail framing) ----------
|
|
259
|
-
|
|
260
|
-
// The instance's project name, injected into the grader's identity line so a
|
|
261
|
-
// non-OTB Cloud Bongos instance is graded against ITS OWN project, never a
|
|
262
|
-
// hardcoded "Off the Boats" (ADR 0062 §3/§5; the branding contract's
|
|
263
|
-
// llmProjectName field). Reached through the doorway like every other host-
|
|
264
|
-
// context read in this module — never a deep core require. Resolved once at
|
|
265
|
-
// module load (the brand is a per-process singleton); falls back to a neutral
|
|
266
|
-
// phrase if branding fails to resolve so the gate never crashes on a config error.
|
|
267
|
-
const LLM_PROJECT = (() => {
|
|
268
|
-
try { return api.branding().llmProjectName || 'this project'; }
|
|
269
|
-
catch { return 'this project'; }
|
|
270
|
-
})();
|
|
271
|
-
|
|
272
|
-
const PROMPT_HEADER = `You are an adversarial code-review subagent for the ${LLM_PROJECT} project. You are reviewing one shipped change to decide whether it deserves to ship.
|
|
273
|
-
|
|
274
|
-
Your job is to FIND WHAT IS WRONG with this change. **The default verdict is FAIL.** To return a PASS, you must affirmatively conclude — after a careful read of the diff and (using your Read tool) the relevant post-merge files — that no material problems remain.
|
|
275
|
-
|
|
276
|
-
Begin by hunting for these specific failure modes. When you find one, CITE the diff line or name the failing acceptance criterion:
|
|
277
|
-
|
|
278
|
-
- TODOs, FIXMEs, "for now", "temporary" or "stub" comments left in shipped code.
|
|
279
|
-
- Hardcoded shortcuts: assuming one builder, one user, one project root, one tenant, one environment, or one identity. Look for literal "lars", "lars589", absolute /Users paths, hardcoded ports, hardcoded IDs.
|
|
280
|
-
- Scope gaps: acceptance criteria in the description that the diff DOES NOT satisfy. Read the description's "Acceptance" section line by line.
|
|
281
|
-
- Demo / dummy / mock code that lies: returns a fixed value, fakes a successful result, mocks at the wrong layer, or pretends to call something it doesn't.
|
|
282
|
-
- Missing tests for new logic: any non-trivial new function, route handler, branch, or aggregation that ships without a covering test.
|
|
283
|
-
- Broken invariants: methodology rules in CLAUDE.md, cross-verified Python/JS autotile lookups, ground-vs-object cell split per ADR 0012, the "every change is backed by a claimed GDS task" rule, the trust-boundary rule per ADR 0016 ("server-enforced permissions"), the no-hardcoded-Archon-identity rule per V3.R12 #236.
|
|
284
|
-
- Dead code: helpers, exports, branches, or files added but never called from anywhere in the post-merge tree. Use Read on the receiving site if unsure.
|
|
285
|
-
- Reads/writes outside the declared touches[]: if the diff edits files the task didn't claim, that's a scope creep — call it out.
|
|
286
|
-
|
|
287
|
-
You return three scores (0-10), notes, an issues array, and a confidence value:
|
|
288
|
-
|
|
289
|
-
1. functional_fidelity (0-10): did the diff implement what the description scoped? 10 = every acceptance criterion satisfied by code or explicitly out-of-scope per the description. 6 = mostly there with one NAMED gap. Below 6 = scope gaps, broken acceptance criteria, or demo code that lies.
|
|
290
|
-
2. code_quality (0-10): is the code well-written for THIS codebase? 10 = production-quality, would-merge-without-comment. 6 = ships but with a clear improvement a reviewer would suggest. Below 6 = broken style, dead code, broken invariants, missing tests where warranted.
|
|
291
|
-
3. economy (0-10, ADVISORY): is the change as small and simple as the task warrants? 10 = minimal and focused. 6 = a bit more than necessary. Below 6 = bloated, speculative abstraction, duplicated logic.
|
|
292
|
-
|
|
293
|
-
**The hard gate** (applied downstream, you do not enforce it yourself; just score honestly): functional_fidelity >= 6 AND code_quality >= 6 AND confidence >= 0.4. Below 6 on either axis is a FAIL. Economy is advisory only.
|
|
294
|
-
|
|
295
|
-
**For every issue you list:** include severity (blocker | major | minor | nit), the file path, and a one-line summary that quotes the offending diff line or names the failing acceptance criterion. Vague summaries like "code quality could be better" are not acceptable — be specific or omit it.
|
|
296
|
-
|
|
297
|
-
**Sanity checks:**
|
|
298
|
-
- A PASS with no issues[] entries is suspicious. If you found nothing material wrong, briefly say WHY in notes_md: which acceptance criteria you verified, which files you Read, which could-have-gone-wrong didn't.
|
|
299
|
-
- A FAIL with no issues[] entries is invalid. At least one concrete issue must explain the verdict.
|
|
300
|
-
- A score below 6 with no issue at severity major or blocker is inconsistent — name the major problem.
|
|
301
|
-
|
|
302
|
-
**Confidence:** 0-1, your self-reported confidence in your two scores. If you cannot judge (description too vague to grade against, diff too large to fit comfortably in context, key context missing), set confidence LOW (< 0.4). Low confidence falls through to manual confirm — be honest about uncertainty rather than guessing.
|
|
303
|
-
|
|
304
|
-
Return ONLY JSON matching this schema — no prose before or after, no markdown fences:
|
|
305
|
-
{
|
|
306
|
-
"functional_fidelity": <number 0-10>,
|
|
307
|
-
"code_quality": <number 0-10>,
|
|
308
|
-
"economy": <number 0-10>,
|
|
309
|
-
"notes_md": <string>,
|
|
310
|
-
"confidence": <number 0-1>,
|
|
311
|
-
"issues": [{"severity": "blocker"|"major"|"minor"|"nit", "file": <path>, "summary": <string>}, ...]
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
You have a Read tool. Spend reads checking that removed code is unreferenced elsewhere, that new helpers match surrounding style, that the diff's value_summary matches what the diff actually does, and that acceptance criteria you can't verify from the diff alone are actually satisfied in the post-merge file. Do not run tests, do not edit files, do not call the network.`;
|
|
315
|
-
|
|
316
|
-
function buildPrompt({ task, valueSummary, diff, changedFiles, diffStats, generatedProvenance: prov = null }) {
|
|
317
|
-
const t = task || {};
|
|
318
|
-
const desc = (t.description || '(no description)').trim();
|
|
319
|
-
const title = (t.title || '(no title)').trim();
|
|
320
|
-
const vs = (valueSummary || '(no value summary)').trim();
|
|
321
|
-
const files = Array.isArray(changedFiles) ? changedFiles : [];
|
|
322
|
-
const diffText = (diff || '(empty diff)').trim();
|
|
323
|
-
|
|
324
|
-
const lines = [];
|
|
325
|
-
lines.push(PROMPT_HEADER);
|
|
326
|
-
lines.push('');
|
|
327
|
-
lines.push('---');
|
|
328
|
-
lines.push('## Task');
|
|
329
|
-
lines.push('');
|
|
330
|
-
lines.push(`**ID:** ${t.id ?? '(unknown)'}`);
|
|
331
|
-
lines.push(`**Title:** ${title}`);
|
|
332
|
-
lines.push(`**Kind:** ${t.kind ?? 'unclassified'} · **Discipline:** ${t.discipline ?? 'unclassified'}`);
|
|
333
|
-
lines.push(`**Estimated effort:** ${t.est_minutes != null ? `${t.est_minutes} min` : '(not estimated)'} — use this to judge whether the diff size is proportionate to the work scoped.`);
|
|
334
|
-
if (t.version_id) lines.push(`**Version:** ${t.version_id}`);
|
|
335
|
-
lines.push('');
|
|
336
|
-
lines.push('**Description:**');
|
|
337
|
-
lines.push('');
|
|
338
|
-
lines.push(desc);
|
|
339
|
-
lines.push('');
|
|
340
|
-
lines.push('---');
|
|
341
|
-
lines.push('## Builder\'s value summary');
|
|
342
|
-
lines.push('');
|
|
343
|
-
lines.push(vs);
|
|
344
|
-
lines.push('');
|
|
345
|
-
lines.push('---');
|
|
346
|
-
lines.push(`## Changed files (${files.length})`);
|
|
347
|
-
lines.push('');
|
|
348
|
-
for (const f of files) lines.push(`- ${f}`);
|
|
349
|
-
if (files.length === 0) lines.push('(no files changed — this is suspicious)');
|
|
350
|
-
lines.push('');
|
|
351
|
-
lines.push('You can Read any of these files to see their post-merge state.');
|
|
352
|
-
lines.push('');
|
|
353
|
-
lines.push('---');
|
|
354
|
-
lines.push('## Diff size (for the economy axis)');
|
|
355
|
-
lines.push('');
|
|
356
|
-
if (diffStats && typeof diffStats === 'object') {
|
|
357
|
-
const fc = diffStats.files_changed ?? '?';
|
|
358
|
-
const la = diffStats.lines_added ?? '?';
|
|
359
|
-
const lr = diffStats.lines_removed ?? '?';
|
|
360
|
-
lines.push(`- files changed: ${fc}`);
|
|
361
|
-
lines.push(`- lines added: ${la}`);
|
|
362
|
-
lines.push(`- lines removed: ${lr}`);
|
|
363
|
-
lines.push('');
|
|
364
|
-
lines.push('Weigh this against the task description, kind, and estimated effort above when scoring economy. A diff much larger than the scoped work warrants — or one padded with speculative abstraction, duplication, or dead code — should score low on economy.');
|
|
365
|
-
} else {
|
|
366
|
-
lines.push('(diff stats unavailable — score economy from the diff itself.)');
|
|
367
|
-
}
|
|
368
|
-
lines.push('');
|
|
369
|
-
lines.push('---');
|
|
370
|
-
lines.push('## Unified diff');
|
|
371
|
-
lines.push('');
|
|
372
|
-
lines.push('```diff');
|
|
373
|
-
lines.push(diffText);
|
|
374
|
-
lines.push('```');
|
|
375
|
-
lines.push('', ...(prov ? [generatedProvenance.provenanceSection(prov)].filter(Boolean) : [])); // task 1002795 — the trivial path is where a docs-only ship lands
|
|
376
|
-
lines.push('Now grade. Default-fail unless your read of the diff and post-merge files turns up no material problems. Output ONLY the JSON object — nothing before or after it.');
|
|
377
|
-
return lines.join('\n');
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// ---------- v3: per-shot validator (same as v2 logic) ----------
|
|
381
|
-
|
|
382
|
-
// Tries to extract a JSON object from a subagent response. The subagent is
|
|
383
|
-
// instructed to return ONLY JSON, but in practice models sometimes wrap it
|
|
384
|
-
// in markdown fences or add a line of prose. Be tolerant.
|
|
385
|
-
// #79 (#380, completed #493): robust JSON extraction now lives in the shared
|
|
386
|
-
// leaf module so grader.js AND every worker module use ONE implementation.
|
|
387
|
-
// Re-exported here for back-compat (validateAndScore + tests use
|
|
388
|
-
// grader.extractJson).
|
|
389
|
-
const { extractJson } = require('./grader-workers/extract-json');
|
|
390
|
-
|
|
391
|
-
function isNum(n) {
|
|
392
|
-
return Number.isFinite(n);
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
// #1002653 corroboration helpers for validateAndScore's null-ff lens branch.
|
|
396
|
-
// A worker entry is untrusted input — w.issues may be any shape. Absent issues
|
|
397
|
-
// on an errored worker is the normal outage shape (treated as none); a truthy
|
|
398
|
-
// NON-ARRAY issues is malformed and must DISQUALIFY corroboration (fail-closed
|
|
399
|
-
// schema_error downstream), never TypeError into a 500 on grade submission.
|
|
400
|
-
const isNarcErroredClean = (w) => {
|
|
401
|
-
if (!w || w.kind !== 'narc' || !w.error) return false;
|
|
402
|
-
const issues = w.issues == null ? [] : w.issues;
|
|
403
|
-
if (!Array.isArray(issues)) return false;
|
|
404
|
-
return !issues.some((i) => i && (i.severity === 'blocker' || i.severity === 'major'));
|
|
405
|
-
};
|
|
406
|
-
const isNarcHealthy = (w) => !!w && w.kind === 'narc' && !w.error;
|
|
407
|
-
|
|
408
|
-
// task 1002658 (ADR 0164): a SUBSTANTIVE issue is one a real panel emits — a
|
|
409
|
-
// known severity + a non-empty summary. The old plausibility guard counted raw
|
|
410
|
-
// issues[].length, so one fabricated `{}` defeated it; substance-counting
|
|
411
|
-
// closes that lazy-forgery class using only the submitted body itself.
|
|
412
|
-
const ISSUE_SEVERITIES = new Set(['blocker', 'major', 'minor', 'nit']);
|
|
413
|
-
function isSubstantiveIssue(it) {
|
|
414
|
-
return !!it && typeof it === 'object'
|
|
415
|
-
&& ISSUE_SEVERITIES.has(String(it.severity || '').toLowerCase())
|
|
416
|
-
&& typeof it.summary === 'string' && it.summary.trim().length > 0;
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
// task 1002658 (ADR 0164): does the submitted signals block carry the full
|
|
420
|
-
// per-worker shape a real panel run produces — ≥3 workers, each with a kind
|
|
421
|
-
// and a legal verdict (or an error tag)? Client-forgeable like every
|
|
422
|
-
// grade-time input (the ADR's documented honor-system residual for Metic+);
|
|
423
|
-
// its job is separating "the panel ran and unanimously found nothing" (the
|
|
424
|
-
// guard's confirmed false-positive class) from "no evidence at all".
|
|
425
|
-
const WORKER_VERDICTS = new Set(['pass', 'concerns', 'fail']);
|
|
426
|
-
function hasPanelEvidence(signals) {
|
|
427
|
-
const ws = Array.isArray(signals?.workers) ? signals.workers : null;
|
|
428
|
-
if (!ws || ws.length < 3) return false;
|
|
429
|
-
return ws.every((w) => !!w && typeof w === 'object'
|
|
430
|
-
&& typeof w.kind === 'string' && w.kind.trim().length > 0
|
|
431
|
-
&& (WORKER_VERDICTS.has(String(w.verdict || '').toLowerCase()) || !!w.error));
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
// Validates ONE shot's subagent JSON and produces a per-shot grade-result.
|
|
435
|
-
// Never throws — malformed inputs produce a passed=false result with notes_md.
|
|
436
|
-
// The downstream aggregateShots collapses N of these into a final verdict.
|
|
437
|
-
function validateAndScore(rawSubagent, meta = {}) {
|
|
438
|
-
const parsed =
|
|
439
|
-
typeof rawSubagent === 'string' ? extractJson(rawSubagent) : rawSubagent;
|
|
440
|
-
|
|
441
|
-
const signals = {
|
|
442
|
-
grader_kind: 'subagent',
|
|
443
|
-
grader_version: VERSION,
|
|
444
|
-
model: meta.model || DEFAULT_GRADER_MODEL,
|
|
445
|
-
threshold: THRESHOLD,
|
|
446
|
-
...meta,
|
|
447
|
-
};
|
|
448
|
-
|
|
449
|
-
if (!parsed || typeof parsed !== 'object') {
|
|
450
|
-
return {
|
|
451
|
-
score: 0,
|
|
452
|
-
passed: false,
|
|
453
|
-
threshold: THRESHOLD,
|
|
454
|
-
grader_kind: 'subagent',
|
|
455
|
-
grader_version: VERSION,
|
|
456
|
-
rubric_scores: {},
|
|
457
|
-
signals: { ...signals, parse_error: true },
|
|
458
|
-
notes_md: `Subagent grader ${VERSION}: could not parse subagent output as JSON.`,
|
|
459
|
-
issues: [],
|
|
460
|
-
};
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
// #1002653: Number(null) === 0, so an errored ADVISORY Narc forwarding an
|
|
464
|
-
// EXPLICIT ff:null fabricated "FAIL — functional_fidelity below 6" at
|
|
465
|
-
// (0+cq)/2. Explicit null = the lens did not run (the #395 confidence
|
|
466
|
-
// precedent): gate/score on the axes that ran. A bare null is NOT enough —
|
|
467
|
-
// a prompt-injected Narc emitting null must stay fail-closed — so the branch
|
|
468
|
-
// needs POSITIVE corroboration: workers[] carries an errored
|
|
469
|
-
// no-substantive-finding Narc (#473 missing-signal semantics) and no healthy
|
|
470
|
-
// Narc. Empty/narc-less/absent workers, ABSENT ff, and every other shape
|
|
471
|
-
// fall through to the fail-closed schema_error below. Quality has no such
|
|
472
|
-
// branch — gating lens, absence stays fail-closed.
|
|
473
|
-
// Trust note: workers[] is client-supplied like every SR-9 grade input — a
|
|
474
|
-
// forger gains nothing here they cannot get by sending ff:9 directly; the
|
|
475
|
-
// server-derivable-signals redesign is tracked separately (SR-9 residual).
|
|
476
|
-
const metaWorkers = Array.isArray(meta.workers) ? meta.workers : null;
|
|
477
|
-
const narcMissingSignal = !!metaWorkers &&
|
|
478
|
-
metaWorkers.some(isNarcErroredClean) && !metaWorkers.some(isNarcHealthy);
|
|
479
|
-
const ffExplicitNull = parsed.functional_fidelity === null;
|
|
480
|
-
const ffLensUnavailable = ffExplicitNull && narcMissingSignal;
|
|
481
|
-
const ff = ffExplicitNull ? null : Number(parsed.functional_fidelity);
|
|
482
|
-
const cq = Number(parsed.code_quality);
|
|
483
|
-
const economyRaw = Number(parsed.economy);
|
|
484
|
-
const economy =
|
|
485
|
-
isNum(economyRaw) && economyRaw >= 0 && economyRaw <= 10 ? economyRaw : null;
|
|
486
|
-
// MAS Phase 2.5 (#423): security_score (from Hacker, Phase 2.1) and
|
|
487
|
-
// efficiency_score (from Efficiency Monger, Phase 2.2). Both ADVISORY —
|
|
488
|
-
// recorded in rubric_scores but do not independently gate the ship. Same
|
|
489
|
-
// back-compat shape as economy: out-of-range / non-numeric / missing →
|
|
490
|
-
// recorded as null, no penalty. Older callers that don't forward these
|
|
491
|
-
// fields grade exactly as before.
|
|
492
|
-
const securityRaw = Number(parsed.security_score);
|
|
493
|
-
const securityScore =
|
|
494
|
-
isNum(securityRaw) && securityRaw >= 0 && securityRaw <= 10 ? securityRaw : null;
|
|
495
|
-
const efficiencyRaw = Number(parsed.efficiency_score);
|
|
496
|
-
const efficiencyScore =
|
|
497
|
-
isNum(efficiencyRaw) && efficiencyRaw >= 0 && efficiencyRaw <= 10 ? efficiencyRaw : null;
|
|
498
|
-
// Treat null/undefined confidence as ABSENT, not as 0. Number(null) === 0,
|
|
499
|
-
// and isNum(0) === true — a naive `isNum(Number(parsed.confidence))` check
|
|
500
|
-
// would accept null as a valid low-confidence signal and FAIL the ship for
|
|
501
|
-
// a missing field. The MAS Phase 1 self-ship (#395) hit this exactly when
|
|
502
|
-
// aggregateShots forwarded a null top-level confidence to the server's
|
|
503
|
-
// validateAndScore. Explicit null-check first; coerce only when present.
|
|
504
|
-
const confidence = parsed.confidence == null
|
|
505
|
-
? null
|
|
506
|
-
: isNum(Number(parsed.confidence)) ? Number(parsed.confidence) : null;
|
|
507
|
-
const issues = Array.isArray(parsed.issues)
|
|
508
|
-
? parsed.issues.filter((it) => it && typeof it === 'object').slice(0, 50)
|
|
509
|
-
: [];
|
|
510
|
-
const notesFromSubagent = typeof parsed.notes_md === 'string' ? parsed.notes_md : '';
|
|
511
|
-
|
|
512
|
-
signals.confidence = confidence;
|
|
513
|
-
signals.issue_count = issues.length;
|
|
514
|
-
if (ffLensUnavailable) signals.ff_lens_unavailable = true;
|
|
515
|
-
|
|
516
|
-
if ((!ffLensUnavailable && (!isNum(ff) || ff < 0 || ff > 10)) || !isNum(cq) || cq < 0 || cq > 10) {
|
|
517
|
-
return {
|
|
518
|
-
score: 0,
|
|
519
|
-
passed: false,
|
|
520
|
-
threshold: THRESHOLD,
|
|
521
|
-
grader_kind: 'subagent',
|
|
522
|
-
grader_version: VERSION,
|
|
523
|
-
rubric_scores: {},
|
|
524
|
-
signals: { ...signals, schema_error: true },
|
|
525
|
-
notes_md: `Subagent grader ${VERSION}: subagent returned scores outside [0,10] or non-numeric. Got functional_fidelity=${parsed.functional_fidelity}, code_quality=${parsed.code_quality}.`
|
|
526
|
-
+ (ffExplicitNull && !narcMissingSignal
|
|
527
|
-
? ' Explicit null functional_fidelity without a corroborated Narc outage in signals.workers is fail-closed.'
|
|
528
|
-
: ''),
|
|
529
|
-
issues,
|
|
530
|
-
};
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
const ffR = ffLensUnavailable ? null : r1(clamp(ff, 0, 10));
|
|
534
|
-
const cqR = r1(clamp(cq, 0, 10));
|
|
535
|
-
const economyR = economy != null ? r1(clamp(economy, 0, 10)) : null;
|
|
536
|
-
const securityR = securityScore != null ? r1(clamp(securityScore, 0, 10)) : null;
|
|
537
|
-
const efficiencyR = efficiencyScore != null ? r1(clamp(efficiencyScore, 0, 10)) : null;
|
|
538
|
-
const economyPenalty =
|
|
539
|
-
economyR != null ? Math.max(0, (THRESHOLD - economyR) / 2) : 0;
|
|
540
|
-
const cqEffective = economyR != null ? r1(clamp(cqR - economyPenalty, 0, 10)) : cqR;
|
|
541
|
-
// With the ff lens out, the score reflects only the axis that ran — never a
|
|
542
|
-
// blend with a synthetic 0 (the fabricated 4.5/10) or a synthetic threshold.
|
|
543
|
-
const blended = ffLensUnavailable ? cqR : r1((ffR + cqR) / 2);
|
|
544
|
-
|
|
545
|
-
// confidence must be present (not null) for a PASS. A subagent that omits
|
|
546
|
-
// confidence entirely silently bypassed the low-confidence gate under the
|
|
547
|
-
// old `confidence != null && confidence < LOW_CONFIDENCE` check — null made
|
|
548
|
-
// lowConfidence=false and let high ff/cq scores through unchecked (#489 / #56).
|
|
549
|
-
// Distinguishing "absent" from "low" gives a clear verdict message and leaves
|
|
550
|
-
// the aggregateShots #395 fix intact: shots that omit confidence now FAIL here,
|
|
551
|
-
// so aggregateShots can never reach unanimousPASS with a null confidenceMedian.
|
|
552
|
-
const confidenceAbsent = confidence === null;
|
|
553
|
-
const lowConfidence = !confidenceAbsent && confidence < LOW_CONFIDENCE;
|
|
554
|
-
// ffR is null when the lens is unavailable — it must not fail the gate
|
|
555
|
-
// (null < THRESHOLD is true via JS coercion), and must not appear in the
|
|
556
|
-
// verdict's below-threshold branches.
|
|
557
|
-
const ffBelow = !ffLensUnavailable && ffR < THRESHOLD;
|
|
558
|
-
const bothPass = !ffBelow && cqR >= THRESHOLD;
|
|
559
|
-
const passed = bothPass && !lowConfidence && !confidenceAbsent;
|
|
560
|
-
|
|
561
|
-
const verdict = passed
|
|
562
|
-
? 'PASS'
|
|
563
|
-
: confidenceAbsent
|
|
564
|
-
? 'FAIL — confidence absent'
|
|
565
|
-
: lowConfidence
|
|
566
|
-
? 'FAIL — low confidence'
|
|
567
|
-
: bothPass
|
|
568
|
-
? 'FAIL'
|
|
569
|
-
: ffBelow && cqR < THRESHOLD
|
|
570
|
-
? `FAIL — both axes below ${THRESHOLD}`
|
|
571
|
-
: ffBelow
|
|
572
|
-
? `FAIL — functional_fidelity below ${THRESHOLD}`
|
|
573
|
-
: `FAIL — code_quality below ${THRESHOLD}`;
|
|
574
|
-
|
|
575
|
-
const notesLines = [];
|
|
576
|
-
notesLines.push(`Shot ${signals.shot_index != null ? signals.shot_index : '?'} (${signals.model}) — ${verdict}`);
|
|
577
|
-
notesLines.push(` ff=${ffLensUnavailable ? 'n/a (lens unavailable)' : `${ffR}/10`} cq=${cqR}/10${economyR != null ? ` econ=${economyR}/10 (advisory)` : ''} conf=${confidence ?? '?'}`);
|
|
578
|
-
if (notesFromSubagent) {
|
|
579
|
-
notesLines.push('');
|
|
580
|
-
notesLines.push(notesFromSubagent.trim());
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
return {
|
|
584
|
-
score: blended,
|
|
585
|
-
passed,
|
|
586
|
-
threshold: THRESHOLD,
|
|
587
|
-
grader_kind: 'subagent',
|
|
588
|
-
grader_version: VERSION,
|
|
589
|
-
rubric_scores: {
|
|
590
|
-
functional_fidelity: ffR,
|
|
591
|
-
code_quality: cqR,
|
|
592
|
-
economy: economyR,
|
|
593
|
-
code_quality_effective: cqEffective,
|
|
594
|
-
security_score: securityR,
|
|
595
|
-
efficiency_score: efficiencyR,
|
|
596
|
-
},
|
|
597
|
-
signals,
|
|
598
|
-
notes_md: notesLines.join('\n'),
|
|
599
|
-
issues,
|
|
600
|
-
};
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
// ---------- server-authoritative grade guards (SR-9 / SR-18 / F1 / G5) ----------
|
|
604
|
-
//
|
|
605
|
-
// THE FINDING (SR-9 / SR-18): POST /api/gds/tasks/:id/grade trusts the client.
|
|
606
|
-
// validateAndScore only shape-validates the numbers — `passed` is computed from
|
|
607
|
-
// the CLIENT's submitted scores. The real adversarial panel + the deterministic
|
|
608
|
-
// route-rank / permission-path pre-passes run CLIENT-SIDE in scripts/gds/ship.js;
|
|
609
|
-
// the server never re-runs them. So a builder can POST
|
|
610
|
-
// {functional_fidelity:10, code_quality:10, confidence:1, issues:[]} to self-pass.
|
|
611
|
-
// AND the credit/achievement side-effects (net-negative "Subtractor" bonus,
|
|
612
|
-
// shipper badges) are driven by client-supplied signals.diff_stats +
|
|
613
|
-
// committed_files the server never re-derives (SR-18).
|
|
614
|
-
//
|
|
615
|
-
// WHAT THE SERVER CAN AND CAN'T DO (the boundary — see report):
|
|
616
|
-
// * The web tier is checked out at `main`. The builder's feature-branch
|
|
617
|
-
// commits are NOT on the server at grade time (they only arrive when the
|
|
618
|
-
// task merges to main). So the server CANNOT independently re-derive the
|
|
619
|
-
// committed diff at the claimed head_sha. The diff, diff_stats, and
|
|
620
|
-
// committed_files in the request body remain CLIENT-SUPPLIED.
|
|
621
|
-
// * BUT the server DOES know the caller's LIVE, authoritative rank
|
|
622
|
-
// (req.builder.rank, read fresh from the DB per request — ADR 0016). So the
|
|
623
|
-
// deterministic permission-path pre-pass can run with a TRUSTED rank against
|
|
624
|
-
// the (client-supplied) committed_files — strictly raising the bar.
|
|
625
|
-
// * The route-rank pre-pass reads the route files in the server's OWN
|
|
626
|
-
// checkout (fs) — fully server-authoritative, independent of any client
|
|
627
|
-
// input. (NB: it audits `main`, not the feature branch — the post-push main
|
|
628
|
-
// audit owns feature-branch route changes. Re-running here still hard-fails
|
|
629
|
-
// a forged pass while `main` already carries an ungated write route.)
|
|
630
|
-
//
|
|
631
|
-
// THE GUARD (additive, fail-toward-reject-only): serverSideGuards re-runs the
|
|
632
|
-
// two deterministic pre-passes server-side and applies plausibility/bounding
|
|
633
|
-
// checks. It can ONLY downgrade a client PASS to a non-pass (hard FAIL or
|
|
634
|
-
// manual-confirm); it NEVER upgrades a fail, never relaxes the existing gate,
|
|
635
|
-
// and is a no-op on a client that already reported not-passed. A legitimate
|
|
636
|
-
// honest grade is unaffected.
|
|
637
|
-
//
|
|
638
|
-
// The Archon / GRADER_BYPASS paths do not flow through /grade's passed-gate the
|
|
639
|
-
// way an ordinary ship does (they confirm via /tasks/:id/confirm, which is a
|
|
640
|
-
// separate route), so this guard does not touch them — see the route comment.
|
|
641
|
-
|
|
642
|
-
// Plausibility threshold: a client PASS that reports NO issues on a NON-TRIVIAL
|
|
643
|
-
// diff is not a credible adversarial-panel verdict — the panel is default-fail
|
|
644
|
-
// and every worker must enumerate concrete issues before a pass; a clean pass on
|
|
645
|
-
// a large diff with an empty issues[] is the exact shape of a forged
|
|
646
|
-
// {ff:10,cq:10,issues:[]} self-pass (SR-9). We don't reject it outright (a small
|
|
647
|
-
// honest change can legitimately have no issues) — we DOWNGRADE it to
|
|
648
|
-
// manual-confirm so a human lands it via /confirm. Thresholds mirror the
|
|
649
|
-
// trivial-diff fast-path's spirit (a change past these sizes is "real work"):
|
|
650
|
-
// - more than this many changed files, OR
|
|
651
|
-
// - more than this many added+removed lines
|
|
652
|
-
// makes an empty-issues PASS implausible-as-automatic and forces manual-confirm.
|
|
653
|
-
const IMPLAUSIBLE_PASS_MAX_FILES = TRIVIAL_MAX_FILES; // 3 — same "small change" bar
|
|
654
|
-
const IMPLAUSIBLE_PASS_MAX_LINES = TRIVIAL_MAX_LINES; // 30
|
|
655
|
-
|
|
656
|
-
// Re-derive the size of the change from the submitted committed_files +
|
|
657
|
-
// diff_stats. Both are client-supplied (see boundary note) — we take the
|
|
658
|
-
// STRICTER (larger) of the file-list length and the shortstat files_changed so a
|
|
659
|
-
// forger can't shrink one to dodge the plausibility bar while the other betrays
|
|
660
|
-
// a large diff. Returns { files, lines } with NaN coerced to 0.
|
|
661
|
-
function changeMagnitude({ committedFiles, diffStats } = {}) {
|
|
662
|
-
const listLen = Array.isArray(committedFiles)
|
|
663
|
-
? committedFiles.filter((f) => typeof f === 'string' && f.trim()).length
|
|
664
|
-
: 0;
|
|
665
|
-
const statFiles = diffStats && Number.isFinite(Number(diffStats.files_changed))
|
|
666
|
-
? Number(diffStats.files_changed)
|
|
667
|
-
: 0;
|
|
668
|
-
const files = Math.max(listLen, statFiles);
|
|
669
|
-
const lines =
|
|
670
|
-
(Number(diffStats?.lines_added) || 0) + (Number(diffStats?.lines_removed) || 0);
|
|
671
|
-
return { files, lines };
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// serverSideGuards — the server-authoritative hardening layer (SR-9 / SR-18).
|
|
675
|
-
//
|
|
676
|
-
// Inputs:
|
|
677
|
-
// gradeResult — the output of validateAndScore on the SUBMITTED body. Its
|
|
678
|
-
// `passed` is computed from the client's numbers. We may
|
|
679
|
-
// override it (downgrade only).
|
|
680
|
-
// committedFiles — client-supplied list of committed paths (or null).
|
|
681
|
-
// builderRank — the caller's LIVE rank (req.builder.rank — SERVER-KNOWN,
|
|
682
|
-
// authoritative). Drives the permission-path pre-pass.
|
|
683
|
-
// diffStats — client-supplied { files_changed, lines_added, lines_removed }
|
|
684
|
-
// (gradeResult.signals.diff_stats if not passed explicitly).
|
|
685
|
-
// routesDir — optional override for tests; defaults to the live routes dir.
|
|
686
|
-
//
|
|
687
|
-
// Returns a NEW grade-result (never mutates the input). When a guard trips, the
|
|
688
|
-
// returned result has passed=false and a `server_guard` block in signals naming
|
|
689
|
-
// every triggered guard + the residual notes. When no guard trips, the input is
|
|
690
|
-
// returned with a passed-through `server_guard: { applied: [...], ok: true }`
|
|
691
|
-
// audit marker so the trail shows the guards RAN (and found nothing) rather than
|
|
692
|
-
// being silently absent.
|
|
693
|
-
//
|
|
694
|
-
// CONTRACT: downgrade-only. If the client already reported not-passed, we leave
|
|
695
|
-
// it not-passed (no resurrection). We only ever flip true→false.
|
|
696
|
-
function serverSideGuards(gradeResult, { committedFiles, builderRank, diffStats, routesDir } = {}) {
|
|
697
|
-
const base = gradeResult || {};
|
|
698
|
-
const ds = diffStats || base.signals?.diff_stats || null;
|
|
699
|
-
const guardsTripped = []; // [{ guard, severity, summary }]
|
|
700
|
-
const extraIssues = []; // surfaced into the result's issues[]
|
|
701
|
-
let downgradeTo = null; // 'fail' | 'concerns' | null
|
|
702
|
-
|
|
703
|
-
// The guards only have teeth on a CLIENT PASS — a non-pass is already
|
|
704
|
-
// blocked/fallen-through by the existing gate, and downgrade-only means we
|
|
705
|
-
// never touch it. We still RUN the deterministic checks (their findings are a
|
|
706
|
-
// useful audit signal even on a fail) but only let them flip the verdict when
|
|
707
|
-
// base.passed === true.
|
|
708
|
-
const clientPassed = base.passed === true;
|
|
709
|
-
|
|
710
|
-
// --- Guard 1: deterministic pre-passes, server-side (SR-9 / F1) ---
|
|
711
|
-
// route-rank: reads the server's own route files (fs) — server-authoritative.
|
|
712
|
-
// permission-path: runs with the TRUSTED live rank against the (client) files.
|
|
713
|
-
// Either hard-fail flips a client PASS to a hard FAIL regardless of submitted
|
|
714
|
-
// scores — this is the core "re-run the deterministic checks server-side"
|
|
715
|
-
// hardening. A non-hardFail (e.g. an unclassified READ route → 'concerns') is
|
|
716
|
-
// recorded but does NOT block here: the client-side panel's Hacker already
|
|
717
|
-
// adjudicates read-route intent, and a server-side downgrade-to-concerns on a
|
|
718
|
-
// read route would be a behavior change with no SR-9 security value.
|
|
719
|
-
let routeRank = null;
|
|
720
|
-
try {
|
|
721
|
-
routeRank = routeRankCheck.checkRouteRanks(routesDir ? { routesDir } : {});
|
|
722
|
-
} catch (_) { routeRank = null; /* fs error → no server-side route finding; client panel + main audit still gate */ }
|
|
723
|
-
const permPath = permissionPathCheck.checkPermissionPaths({ changedFiles: committedFiles || [], builderRank });
|
|
724
|
-
|
|
725
|
-
if (routeRank && routeRank.hardFail) {
|
|
726
|
-
guardsTripped.push({ guard: 'route_rank', severity: 'blocker',
|
|
727
|
-
summary: 'server-side route-rank pre-pass hard-failed (an ungated write route exists) — a client PASS cannot override this (SR-9)' });
|
|
728
|
-
for (const f of routeRank.findings.filter((x) => x.severity === 'blocker')) {
|
|
729
|
-
extraIssues.push({ severity: 'blocker', file: f.file, summary: `[server route-rank] ${f.summary}`, worker: 'server-route-rank' });
|
|
730
|
-
}
|
|
731
|
-
if (clientPassed) downgradeTo = 'fail';
|
|
732
|
-
}
|
|
733
|
-
if (permPath && permPath.hardFail) {
|
|
734
|
-
guardsTripped.push({ guard: 'permission_path', severity: 'blocker',
|
|
735
|
-
summary: `server-side permission-path pre-pass hard-failed: rank '${builderRank}' is below Metic and the submitted committed_files touch protected machinery — a client PASS cannot override this (SR-9 / SEC #863)` });
|
|
736
|
-
for (const f of permPath.findings) {
|
|
737
|
-
extraIssues.push({ severity: 'blocker', file: f.file, summary: `[server permission-path] ${f.summary}`, worker: 'server-permission-path' });
|
|
738
|
-
}
|
|
739
|
-
if (clientPassed) downgradeTo = 'fail';
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
// --- Guard 1b: vacuous permission-path input (task 1002646 / report 1000021 G-1) ---
|
|
743
|
-
// The pre-pass above pairs the SERVER-known rank with CLIENT-supplied
|
|
744
|
-
// committed_files, so omitting the list (or sending []) made the sub-floor
|
|
745
|
-
// protected-path check pass VACUOUSLY — zero files, zero hits, hardFail:false —
|
|
746
|
-
// a trusted-rank check applied to an untrusted, omittable diff. The server
|
|
747
|
-
// still cannot re-derive the feature-branch diff at grade time (boundary note
|
|
748
|
-
// above; the task-1002645 published tip is recorded at ci-land, AFTER the
|
|
749
|
-
// grade), but it can refuse to treat "no input" as "no hits": a caller below
|
|
750
|
-
// the registry's lowest floor submitting NO file list makes the check
|
|
751
|
-
// UNPROVABLE → fail closed. Honest ships are unaffected (ship.js always sends
|
|
752
|
-
// the real committed list — task 878 — and a floor-clearing rank never enters
|
|
753
|
-
// this branch). The dishonest-LIST case (a sub-floor caller sending an
|
|
754
|
-
// innocuous list while touching protected paths) stays owned by gate-review CI
|
|
755
|
-
// + the post-push main audit, which read the REAL diff. Unknown rank keeps
|
|
756
|
-
// failing open at this layer (isBelowFloor → false) — the main audit reads the
|
|
757
|
-
// real author rank from the DB and owns that case, exactly as for the pre-pass.
|
|
758
|
-
const submittedFileCount = Array.isArray(committedFiles)
|
|
759
|
-
? committedFiles.filter((f) => typeof f === 'string' && f.trim()).length
|
|
760
|
-
: 0;
|
|
761
|
-
if (permissionPathCheck.isBelowFloor(builderRank) && submittedFileCount === 0) {
|
|
762
|
-
// Resolve the floor label from the registry at runtime (the D-16/N-3
|
|
763
|
-
// stale-string rule) — never a hardcoded rank name.
|
|
764
|
-
const floorRank = Object.keys(permissionPathCheck.RANK_TIER)
|
|
765
|
-
.find((r) => permissionPathCheck.RANK_TIER[r] === permissionPathCheck.MIN_TIER) || 'the floor';
|
|
766
|
-
guardsTripped.push({ guard: 'permission_path_unprovable', severity: 'blocker',
|
|
767
|
-
summary: `rank '${builderRank}' is below the protected-path floor ('${floorRank}') and the grade submitted no committed_files — the permission-path check cannot run on an empty list, so a client PASS cannot stand (fail closed; task 1002646 / SR-9). Remedy: re-run the ship — ship.js submits the real committed file list; a genuinely file-less (DB-only) change needs a ${floorRank}+ builder to land it via /tasks/:id/confirm.` });
|
|
768
|
-
extraIssues.push({ severity: 'blocker', file: '(grade)', summary:
|
|
769
|
-
`[server permission-path] no committed_files submitted by a rank below the '${floorRank}' floor — the protected-path check is unprovable and fails closed (task 1002646)`, worker: 'server-permission-path' });
|
|
770
|
-
if (clientPassed) downgradeTo = 'fail';
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
// --- Guard 2: implausible client verdict (SR-9 / task 1002658, ADR 0164) ---
|
|
774
|
-
// Redesigned around what the submitted body itself can prove. The old form
|
|
775
|
-
// (raw issues.length === 0) was defeated by one fabricated `{}` AND fired on
|
|
776
|
-
// every honest unanimous zero-issue panel PASS (its only in-sample firings —
|
|
777
|
-
// eval 2-for-2, tasks 1002643 ×2 and 1002663 — were ALL false positives).
|
|
778
|
-
// Now: count only SUBSTANTIVE issues; a padded issues[] (entries but no
|
|
779
|
-
// substance) is the forgery signature and trips regardless of evidence; a
|
|
780
|
-
// genuinely empty issues[] trips only when the body ALSO lacks the full
|
|
781
|
-
// per-worker panel shape — a real panel that ran and unanimously found
|
|
782
|
-
// nothing is waived with an audit marker instead of a manual-confirm. The
|
|
783
|
-
// magnitude inputs remain client-supplied; the binding authority on the real
|
|
784
|
-
// diff is land-time (gate-review CI + post-push main audit) — ADR 0164.
|
|
785
|
-
const rawIssueCount = Array.isArray(base.issues) ? base.issues.length : 0;
|
|
786
|
-
const substantiveIssueCount = (Array.isArray(base.issues) ? base.issues : [])
|
|
787
|
-
.filter(isSubstantiveIssue).length;
|
|
788
|
-
const paddedIssues = rawIssueCount > 0 && substantiveIssueCount === 0;
|
|
789
|
-
const panelEvidence = hasPanelEvidence(base.signals);
|
|
790
|
-
const { files: changeFiles, lines: changeLines } = changeMagnitude({ committedFiles, diffStats: ds });
|
|
791
|
-
const nonTrivial = changeFiles > IMPLAUSIBLE_PASS_MAX_FILES || changeLines > IMPLAUSIBLE_PASS_MAX_LINES;
|
|
792
|
-
let implausiblePassWaived = null;
|
|
793
|
-
if (clientPassed && nonTrivial && substantiveIssueCount === 0 && (paddedIssues || !panelEvidence)) {
|
|
794
|
-
const shape = paddedIssues
|
|
795
|
-
? `issues[] holds ${rawIssueCount} entr${rawIssueCount === 1 ? 'y' : 'ies'} with no substance (no known severity + summary) — the padded-issues forgery signature`
|
|
796
|
-
: 'empty issues[] with no per-worker panel evidence in signals';
|
|
797
|
-
guardsTripped.push({ guard: 'implausible_pass', severity: 'major',
|
|
798
|
-
summary: `client PASS on a non-trivial diff (${changeFiles} file(s), ${changeLines} line(s) > ${IMPLAUSIBLE_PASS_MAX_FILES}/${IMPLAUSIBLE_PASS_MAX_LINES}): ${shape} — not a credible default-fail panel verdict → manual-confirm (SR-9 / ADR 0164)` });
|
|
799
|
-
extraIssues.push({ severity: 'major', file: '(grade)', summary:
|
|
800
|
-
`[server plausibility] a passing grade with no substantive findings and no panel evidence on a ${changeFiles}-file / ${changeLines}-line change must be human-confirmed via /confirm`, worker: 'server-plausibility' });
|
|
801
|
-
if (downgradeTo !== 'fail') downgradeTo = 'concerns';
|
|
802
|
-
} else if (clientPassed && nonTrivial && rawIssueCount === 0 && panelEvidence) {
|
|
803
|
-
implausiblePassWaived = 'panel_evidence';
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
// --- Guard 3: bound the credit/achievement signals (SR-18) ---
|
|
807
|
-
// The net-negative "Subtractor" bonus + shipper badges key off client-supplied
|
|
808
|
-
// signals.diff_stats + committed_files the server can't re-derive. We can't
|
|
809
|
-
// make them server-authoritative without the feature-branch diff (residual,
|
|
810
|
-
// see report), but we CAN strip an OBVIOUSLY-FORGED net-negative claim:
|
|
811
|
-
// diff_stats that claims a net-negative (removed > added) while the submitted
|
|
812
|
-
// committed_files list is large is internally inconsistent with a real
|
|
813
|
-
// simplification, and is the cheap way to farm the Subtractor bonus. When the
|
|
814
|
-
// signals are internally implausible we NEUTRALIZE the net-negative reward by
|
|
815
|
-
// zeroing lines_removed in the forwarded diff_stats (so awardNetNegativeBonus
|
|
816
|
-
// sees removed<=added → bonus 0) WITHOUT altering the displayed/audited stats.
|
|
817
|
-
// Conservative: only fires on the inconsistent shape; an honest net-negative
|
|
818
|
-
// simplification (few files, removed>added) is untouched.
|
|
819
|
-
let sanitizedSignals = base.signals;
|
|
820
|
-
const claimsNetNegative =
|
|
821
|
-
ds && (Number(ds.lines_removed) || 0) > (Number(ds.lines_added) || 0);
|
|
822
|
-
const NET_NEGATIVE_MAX_FILES = 25; // a "simplification" touching >25 files is implausibly broad
|
|
823
|
-
if (claimsNetNegative && changeFiles > NET_NEGATIVE_MAX_FILES) {
|
|
824
|
-
guardsTripped.push({ guard: 'forged_net_negative', severity: 'minor',
|
|
825
|
-
summary: `net-negative diff_stats (removed ${ds.lines_removed} > added ${ds.lines_added}) across ${changeFiles} files is implausibly broad for a simplification — net-negative bonus neutralized (SR-18)` });
|
|
826
|
-
// Clone signals + zero the credit-driving field WITHOUT mutating the input
|
|
827
|
-
// or the displayed diff_stats audit value (we add a sibling marker instead).
|
|
828
|
-
sanitizedSignals = {
|
|
829
|
-
...(base.signals || {}),
|
|
830
|
-
diff_stats: { ...ds, lines_removed: Number(ds.lines_added) || 0 },
|
|
831
|
-
diff_stats_reported: ds, // preserve what the client claimed for the audit trail
|
|
832
|
-
};
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
// No guard tripped (or only audit-level findings on a non-pass) → return the
|
|
836
|
-
// input with a passed-through audit marker so the trail shows guards ran.
|
|
837
|
-
if (downgradeTo == null && guardsTripped.length === 0 && sanitizedSignals === base.signals) {
|
|
838
|
-
return {
|
|
839
|
-
...base,
|
|
840
|
-
signals: {
|
|
841
|
-
...(base.signals || {}),
|
|
842
|
-
server_guard: {
|
|
843
|
-
applied: ['route_rank', 'permission_path', 'permission_path_unprovable', 'implausible_pass', 'forged_net_negative'],
|
|
844
|
-
ok: true,
|
|
845
|
-
tripped: [],
|
|
846
|
-
// task 1002658 / ADR 0164: the audit trail records WHY a zero-issue
|
|
847
|
-
// pass on a non-trivial diff stood (panel evidence), so /grade-audit
|
|
848
|
-
// can distinguish waived from never-evaluated.
|
|
849
|
-
...(implausiblePassWaived ? { implausible_pass_waived: implausiblePassWaived } : {}),
|
|
850
|
-
},
|
|
851
|
-
},
|
|
852
|
-
};
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
// Build the downgraded result. Downgrade-only: if base wasn't passing we keep
|
|
856
|
-
// it not-passing; we never resurrect. Map the panel verdict onto the v3
|
|
857
|
-
// server gate the same way aggregatePanel does (fail → confidence 0; concerns
|
|
858
|
-
// → FALLTHROUGH_CONFIDENCE below the low-confidence cap; pass left as-is).
|
|
859
|
-
const newPassed = clientPassed && downgradeTo == null;
|
|
860
|
-
const forwardedConfidence =
|
|
861
|
-
downgradeTo === 'fail'
|
|
862
|
-
? 0
|
|
863
|
-
: downgradeTo === 'concerns'
|
|
864
|
-
? FALLTHROUGH_CONFIDENCE
|
|
865
|
-
: (base.signals?.confidence ?? null);
|
|
866
|
-
|
|
867
|
-
const guardNote = guardsTripped.length
|
|
868
|
-
? `\n\nServer-authoritative guards (SR-9 / SR-18) ${downgradeTo ? `DOWNGRADED this grade to ${downgradeTo.toUpperCase()}` : 'flagged'}:\n`
|
|
869
|
-
+ guardsTripped.map((g) => ` [${g.severity}] ${g.guard}: ${g.summary}`).join('\n')
|
|
870
|
-
: '';
|
|
871
|
-
|
|
872
|
-
return {
|
|
873
|
-
...base,
|
|
874
|
-
passed: newPassed,
|
|
875
|
-
issues: [...(base.issues || []), ...extraIssues],
|
|
876
|
-
signals: {
|
|
877
|
-
...sanitizedSignals,
|
|
878
|
-
// confidence drives the server-side gate alignment (validateAndScore on the
|
|
879
|
-
// body already ran; this is the authoritative post-hoc adjustment).
|
|
880
|
-
confidence: forwardedConfidence,
|
|
881
|
-
server_guard: {
|
|
882
|
-
applied: ['route_rank', 'permission_path', 'permission_path_unprovable', 'implausible_pass', 'forged_net_negative'],
|
|
883
|
-
ok: guardsTripped.length === 0,
|
|
884
|
-
downgraded_to: downgradeTo,
|
|
885
|
-
client_passed: clientPassed,
|
|
886
|
-
tripped: guardsTripped,
|
|
887
|
-
// task 1002614 (map P-2): a permission_path trip is an AUTHORIZATION
|
|
888
|
-
// refusal of the builder's rank over protected paths — not a verdict on
|
|
889
|
-
// the code. Named here so clients render it as a wall, not a quality
|
|
890
|
-
// FAIL the builder should fix-and-retry (the envelope used to teach
|
|
891
|
-
// exactly that wrong lesson). route_rank stays a code finding.
|
|
892
|
-
// task 1002646: permission_path_unprovable is the same wall — the rank
|
|
893
|
-
// floor refusing an unverifiable diff — so it renders the same way.
|
|
894
|
-
authorization_refusal: guardsTripped.some((g) => g.guard === 'permission_path' || g.guard === 'permission_path_unprovable'),
|
|
895
|
-
...(implausiblePassWaived ? { implausible_pass_waived: implausiblePassWaived } : {}),
|
|
896
|
-
},
|
|
897
|
-
},
|
|
898
|
-
notes_md: `${base.notes_md || ''}${guardNote}`,
|
|
899
|
-
};
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
// ---------- v3: multi-shot aggregation ----------
|
|
903
|
-
|
|
904
|
-
// Collapses N per-shot grade-results into the final v3 grade-result.
|
|
905
|
-
// Pass requires unanimous PASS on all shots; anything else (any FAIL, any
|
|
906
|
-
// error, any disagreement) → not passed. Issues are deduped across shots
|
|
907
|
-
// by (file, summary). Displayed rubric_scores use the median across shots
|
|
908
|
-
// so the trend dashboard stays one-number-per-task.
|
|
909
|
-
function aggregateShots(shotResults, meta = {}) {
|
|
910
|
-
const N = shotResults.length;
|
|
911
|
-
const verdicts = shotResults.map((s) => ({
|
|
912
|
-
passed: s.passed,
|
|
913
|
-
ff: s.rubric_scores?.functional_fidelity ?? null,
|
|
914
|
-
cq: s.rubric_scores?.code_quality ?? null,
|
|
915
|
-
economy: s.rubric_scores?.economy ?? null,
|
|
916
|
-
cq_effective: s.rubric_scores?.code_quality_effective ?? null,
|
|
917
|
-
confidence: s.signals?.confidence ?? null,
|
|
918
|
-
model: s.signals?.model ?? null,
|
|
919
|
-
run_ms: s.signals?.run_ms ?? null,
|
|
920
|
-
shot_index: s.signals?.shot_index ?? null,
|
|
921
|
-
notes_md: s.notes_md,
|
|
922
|
-
issues: s.issues || [],
|
|
923
|
-
error:
|
|
924
|
-
s.signals?.parse_error
|
|
925
|
-
? 'parse_error'
|
|
926
|
-
: s.signals?.schema_error
|
|
927
|
-
? 'schema_error'
|
|
928
|
-
: s.signals?.subprocess_error || null,
|
|
929
|
-
}));
|
|
930
|
-
|
|
931
|
-
const ff = median(verdicts.map((v) => v.ff));
|
|
932
|
-
const cq = median(verdicts.map((v) => v.cq));
|
|
933
|
-
const economy = median(verdicts.map((v) => v.economy));
|
|
934
|
-
const cqEffective = median(verdicts.map((v) => v.cq_effective));
|
|
935
|
-
const blended = ff != null && cq != null ? r1((ff + cq) / 2) : 0;
|
|
936
|
-
// Top-level aggregated confidence = median of per-shot confidences. Without
|
|
937
|
-
// this, ship.js forwards a null top-level confidence to /tasks/:id/grade,
|
|
938
|
-
// and the server's validateAndScore (re-running on the POST body) hits the
|
|
939
|
-
// Number(null)=0 trap → FAIL even though every shot independently passed.
|
|
940
|
-
// Caught on the MAS Phase 1 self-ship (#395).
|
|
941
|
-
//
|
|
942
|
-
// Computed UNROUNDED (no r1()) — confidence sits near a 0.4 gate cap so
|
|
943
|
-
// rounding to one decimal could nudge a borderline value across the cap
|
|
944
|
-
// in either direction. Per-shot confidences are already 0-1 floats.
|
|
945
|
-
const confVals = verdicts
|
|
946
|
-
.map((v) => v.confidence)
|
|
947
|
-
.filter((n) => Number.isFinite(n))
|
|
948
|
-
.sort((a, b) => a - b);
|
|
949
|
-
const confidenceMedian = confVals.length === 0
|
|
950
|
-
? null
|
|
951
|
-
: confVals.length % 2
|
|
952
|
-
? confVals[(confVals.length - 1) / 2]
|
|
953
|
-
: (confVals[confVals.length / 2 - 1] + confVals[confVals.length / 2]) / 2;
|
|
954
|
-
|
|
955
|
-
// Dedupe issues across shots by (file, summary). Keep first-seen severity.
|
|
956
|
-
const seen = new Set();
|
|
957
|
-
const issues = [];
|
|
958
|
-
for (const v of verdicts) {
|
|
959
|
-
for (const it of v.issues) {
|
|
960
|
-
if (!it || typeof it !== 'object') continue;
|
|
961
|
-
const key = `${it.file ?? ''}::${(it.summary ?? '').trim()}`;
|
|
962
|
-
if (seen.has(key)) continue;
|
|
963
|
-
seen.add(key);
|
|
964
|
-
issues.push(it);
|
|
965
|
-
}
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
const passCount = verdicts.filter((v) => v.passed === true).length;
|
|
969
|
-
const failCount = verdicts.filter((v) => v.passed === false && !v.error).length;
|
|
970
|
-
const errorCount = verdicts.filter((v) => v.error).length;
|
|
971
|
-
const allPassed = N > 0 && passCount === N;
|
|
972
|
-
|
|
973
|
-
let verdict;
|
|
974
|
-
let passed;
|
|
975
|
-
let aggregationKind;
|
|
976
|
-
if (errorCount > 0) {
|
|
977
|
-
passed = false;
|
|
978
|
-
verdict = `FAIL — ${errorCount}/${N} shot(s) errored`;
|
|
979
|
-
aggregationKind = 'errored';
|
|
980
|
-
} else if (allPassed) {
|
|
981
|
-
passed = true;
|
|
982
|
-
verdict = `PASS — ${N}/${N} shots unanimous`;
|
|
983
|
-
aggregationKind = 'unanimous_pass';
|
|
984
|
-
} else if (passCount === 0) {
|
|
985
|
-
passed = false;
|
|
986
|
-
verdict = `FAIL — ${failCount}/${N} shots`;
|
|
987
|
-
aggregationKind = 'unanimous_fail';
|
|
988
|
-
} else {
|
|
989
|
-
passed = false;
|
|
990
|
-
verdict = `FAIL — ${passCount} pass, ${failCount} fail (no unanimous pass)`;
|
|
991
|
-
aggregationKind = 'disagreement';
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
const lines = [];
|
|
995
|
-
lines.push(`Subagent grader ${VERSION} (multi-shot, ${N}× shots) — ${verdict}`);
|
|
996
|
-
lines.push(` functional_fidelity (median): ${ff ?? '?'}/10`);
|
|
997
|
-
lines.push(` code_quality (median): ${cq ?? '?'}/10`);
|
|
998
|
-
if (economy != null) {
|
|
999
|
-
lines.push(` economy (advisory, median): ${economy}/10`);
|
|
1000
|
-
}
|
|
1001
|
-
lines.push(` blended (median): ${blended}/10 (threshold ${THRESHOLD}, both axes must clear on EVERY shot)`);
|
|
1002
|
-
lines.push('');
|
|
1003
|
-
lines.push('Per-shot:');
|
|
1004
|
-
for (let i = 0; i < verdicts.length; i++) {
|
|
1005
|
-
const v = verdicts[i];
|
|
1006
|
-
const status = v.error ? 'ERROR' : v.passed === true ? 'PASS' : 'FAIL';
|
|
1007
|
-
const ffStr = v.ff != null ? `${v.ff}` : '?';
|
|
1008
|
-
const cqStr = v.cq != null ? `${v.cq}` : '?';
|
|
1009
|
-
const confStr = v.confidence != null ? ` conf=${v.confidence}` : '';
|
|
1010
|
-
const errStr = v.error ? ` (${v.error})` : '';
|
|
1011
|
-
lines.push(` shot[${i}] ${status} ff=${ffStr} cq=${cqStr}${confStr} ${v.model ?? '?'}${errStr}`);
|
|
1012
|
-
}
|
|
1013
|
-
if (issues.length > 0) {
|
|
1014
|
-
lines.push('');
|
|
1015
|
-
lines.push(`Issues (${issues.length} unique across shots):`);
|
|
1016
|
-
for (const it of issues.slice(0, 20)) {
|
|
1017
|
-
const sev = String(it.severity || '?').padEnd(7);
|
|
1018
|
-
const file = String(it.file || '?');
|
|
1019
|
-
const summary = String(it.summary || '').replace(/\s+/g, ' ').trim();
|
|
1020
|
-
lines.push(` [${sev}] ${file} — ${summary}`);
|
|
1021
|
-
}
|
|
1022
|
-
if (issues.length > 20) {
|
|
1023
|
-
lines.push(` ... ${issues.length - 20} more (full set in signals.shots[].issues)`);
|
|
1024
|
-
}
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
// #486 (ADR 0028 path 1): sum each shot's own subprocess cost (envelope
|
|
1028
|
-
// total_cost_usd) so the trivial-diff path reports panel_cost_usd the same way
|
|
1029
|
-
// the full panel does. The single-shot trivial path is the common case here.
|
|
1030
|
-
const panelCostUsd = shotResults.reduce((s, r) => s + (Number(r.cost_usd) || 0), 0);
|
|
1031
|
-
|
|
1032
|
-
return {
|
|
1033
|
-
score: blended,
|
|
1034
|
-
passed,
|
|
1035
|
-
threshold: THRESHOLD,
|
|
1036
|
-
grader_kind: 'subagent-multishot',
|
|
1037
|
-
grader_version: VERSION,
|
|
1038
|
-
panel_cost_usd: panelCostUsd, // #486: panel's own API spend (USD)
|
|
1039
|
-
rubric_scores: {
|
|
1040
|
-
functional_fidelity: ff,
|
|
1041
|
-
code_quality: cq,
|
|
1042
|
-
economy,
|
|
1043
|
-
code_quality_effective: cqEffective,
|
|
1044
|
-
},
|
|
1045
|
-
signals: {
|
|
1046
|
-
grader_kind: 'subagent-multishot',
|
|
1047
|
-
grader_version: VERSION,
|
|
1048
|
-
threshold: THRESHOLD,
|
|
1049
|
-
// Top-level confidence = median across shots. ship.js forwards this to
|
|
1050
|
-
// POST /tasks/:id/grade so the server-side re-validation has a real
|
|
1051
|
-
// number rather than null (see Number(null)=0 fix in validateAndScore).
|
|
1052
|
-
confidence: confidenceMedian,
|
|
1053
|
-
shots: verdicts,
|
|
1054
|
-
shot_count: N,
|
|
1055
|
-
pass_count: passCount,
|
|
1056
|
-
fail_count: failCount,
|
|
1057
|
-
error_count: errorCount,
|
|
1058
|
-
shot_aggregation: aggregationKind,
|
|
1059
|
-
...meta,
|
|
1060
|
-
},
|
|
1061
|
-
notes_md: lines.join('\n'),
|
|
1062
|
-
issues,
|
|
1063
|
-
};
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
|
-
// ---------- v3: subagent subprocess invoker ----------
|
|
1067
|
-
|
|
1068
|
-
// Unchanged from v2 — spawns `claude -p --print --model <model> --output-
|
|
1069
|
-
// format json` with stdin = prompt. The v3 caller varies `model` per shot
|
|
1070
|
-
// via opts.model (set by gradeSubagent below).
|
|
1071
|
-
function runSubagent({ prompt, opts = {} }) {
|
|
1072
|
-
return new Promise((resolve, reject) => {
|
|
1073
|
-
const timeoutMs = opts.timeoutMs ?? 240_000;
|
|
1074
|
-
const cwd = opts.cwd ?? process.cwd();
|
|
1075
|
-
const model = opts.model ?? DEFAULT_GRADER_MODEL;
|
|
1076
|
-
// Env escape hatch (OTB_CLAUDE_BIN / CLAUDE_BIN) for installs where the CLI
|
|
1077
|
-
// isn't named `claude` on PATH — covers any OS, not just Windows.
|
|
1078
|
-
const bin = opts.claudeBin ?? process.env.OTB_CLAUDE_BIN ?? process.env.CLAUDE_BIN ?? 'claude';
|
|
1079
|
-
|
|
1080
|
-
const args = [
|
|
1081
|
-
'-p',
|
|
1082
|
-
'--print',
|
|
1083
|
-
'--model', model,
|
|
1084
|
-
'--output-format', 'json',
|
|
1085
|
-
'--tools', 'Read',
|
|
1086
|
-
'--disable-slash-commands',
|
|
1087
|
-
'--permission-mode', 'bypassPermissions',
|
|
1088
|
-
];
|
|
1089
|
-
|
|
1090
|
-
const startedAt = Date.now();
|
|
1091
|
-
const child = spawn(bin, args, {
|
|
1092
|
-
cwd,
|
|
1093
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
1094
|
-
// On Windows the `claude` CLI is a `claude.cmd` shim that Node's spawn
|
|
1095
|
-
// can't launch without a shell (ENOENT -> SUBAGENT_SPAWN_FAILED, observed
|
|
1096
|
-
// on #662). Route through cmd.exe on win32 only; Mac/Linux keep
|
|
1097
|
-
// shell:false (the default, and the proven-working path). args carry no
|
|
1098
|
-
// user input (the prompt goes in via stdin), so shell injection is a
|
|
1099
|
-
// non-issue. windowsHide suppresses the transient console window.
|
|
1100
|
-
shell: process.platform === 'win32',
|
|
1101
|
-
windowsHide: true,
|
|
1102
|
-
// MAS Phase 3 (#427): mark this child as a subagent so the Conductor
|
|
1103
|
-
// UserPromptSubmit hook (.claude/hooks/conductor.js) bails instead of
|
|
1104
|
-
// injecting advisory context into the grader worker's prompt. Any
|
|
1105
|
-
// subagent we spawn inherits this flag and is likewise suppressed.
|
|
1106
|
-
env: { ...process.env, OTB_SUBAGENT: '1' },
|
|
1107
|
-
});
|
|
1108
|
-
|
|
1109
|
-
let stdout = '';
|
|
1110
|
-
let stderr = '';
|
|
1111
|
-
let timedOut = false;
|
|
1112
|
-
|
|
1113
|
-
const killTimer = setTimeout(() => {
|
|
1114
|
-
timedOut = true;
|
|
1115
|
-
try { child.kill('SIGTERM'); } catch (_) {}
|
|
1116
|
-
setTimeout(() => { try { child.kill('SIGKILL'); } catch (_) {} }, 5_000);
|
|
1117
|
-
}, timeoutMs);
|
|
1118
|
-
|
|
1119
|
-
child.stdout.on('data', (b) => { stdout += b.toString('utf8'); });
|
|
1120
|
-
child.stderr.on('data', (b) => { stderr += b.toString('utf8'); });
|
|
1121
|
-
|
|
1122
|
-
child.on('error', (err) => {
|
|
1123
|
-
clearTimeout(killTimer);
|
|
1124
|
-
reject(Object.assign(new Error(`subagent spawn failed: ${err.message}`), {
|
|
1125
|
-
code: 'SUBAGENT_SPAWN_FAILED',
|
|
1126
|
-
cause: err,
|
|
1127
|
-
}));
|
|
1128
|
-
});
|
|
1129
|
-
|
|
1130
|
-
child.on('close', (exitCode) => {
|
|
1131
|
-
clearTimeout(killTimer);
|
|
1132
|
-
const runMs = Date.now() - startedAt;
|
|
1133
|
-
if (timedOut) {
|
|
1134
|
-
return reject(Object.assign(new Error(`subagent timed out after ${timeoutMs}ms`), {
|
|
1135
|
-
code: 'SUBAGENT_TIMEOUT',
|
|
1136
|
-
run_ms: runMs,
|
|
1137
|
-
stdout,
|
|
1138
|
-
stderr,
|
|
1139
|
-
}));
|
|
1140
|
-
}
|
|
1141
|
-
if (exitCode !== 0) {
|
|
1142
|
-
return reject(Object.assign(new Error(`subagent exited with code ${exitCode}`), {
|
|
1143
|
-
code: 'SUBAGENT_NONZERO_EXIT',
|
|
1144
|
-
exit_code: exitCode,
|
|
1145
|
-
run_ms: runMs,
|
|
1146
|
-
stdout,
|
|
1147
|
-
stderr,
|
|
1148
|
-
}));
|
|
1149
|
-
}
|
|
1150
|
-
resolve({ stdout, stderr, run_ms: runMs, exit_code: exitCode, cost_usd: extractCostUsd(stdout) });
|
|
1151
|
-
});
|
|
1152
|
-
|
|
1153
|
-
child.stdin.write(prompt, 'utf8');
|
|
1154
|
-
child.stdin.end();
|
|
1155
|
-
});
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
// ---------- task 1002660: shared context bundle ----------
|
|
1159
|
-
//
|
|
1160
|
-
// ~90% of panel cost was fixed re-exploration: four subprocesses each spending
|
|
1161
|
-
// agentic Read turns re-opening the same changed files. Read them ONCE here
|
|
1162
|
-
// and hand every worker the post-change contents inline (fenced, untrusted),
|
|
1163
|
-
// so a worker's Read tool is only needed for files beyond the bundle.
|
|
1164
|
-
const CONTEXT_BUNDLE_MAX_FILES = 24;
|
|
1165
|
-
const CONTEXT_BUNDLE_MAX_TOTAL_CHARS = 120_000;
|
|
1166
|
-
const CONTEXT_BUNDLE_MAX_FILE_CHARS = 16_000;
|
|
1167
|
-
// Binary-ish extensions never worth inlining (the diff already names them).
|
|
1168
|
-
const CONTEXT_BUNDLE_SKIP_RE = /\.(?:png|jpe?g|gif|webp|ico|svgz|woff2?|ttf|eot|zip|gz|tgz|tar|pdf|mp[34]|wav|ogg|bin|exe|dll|node)$/i;
|
|
1169
|
-
|
|
1170
|
-
// Added lines per file, from the unified diff. Only lines long enough to be a
|
|
1171
|
-
// meaningful fingerprint (a bare `}` or `});` matches anywhere) — used to prove
|
|
1172
|
-
// a truncated bundle prefix actually SHOWS the change (task 1002905).
|
|
1173
|
-
const BUNDLE_EVIDENCE_MIN_CHARS = 12;
|
|
1174
|
-
function diffAddedLinesByFile(diff) {
|
|
1175
|
-
const byFile = new Map();
|
|
1176
|
-
let cur = null;
|
|
1177
|
-
let prevWasOldHeader = false;
|
|
1178
|
-
for (const line of String(diff || '').split('\n')) {
|
|
1179
|
-
// A '+++ ' header only counts right after its '--- ' partner: a content line
|
|
1180
|
-
// reading '++ x' arrives as '+++ x' and must not be mistaken for a header.
|
|
1181
|
-
if (prevWasOldHeader && line.startsWith('+++ ')) {
|
|
1182
|
-
prevWasOldHeader = false;
|
|
1183
|
-
const p = line.slice(4).trim().replace(/^b\//, '');
|
|
1184
|
-
cur = p === '/dev/null' ? null : p;
|
|
1185
|
-
continue;
|
|
1186
|
-
}
|
|
1187
|
-
prevWasOldHeader = line.startsWith('--- ');
|
|
1188
|
-
if (prevWasOldHeader || line.startsWith('diff --git')) continue;
|
|
1189
|
-
if (!cur || !line.startsWith('+')) continue;
|
|
1190
|
-
const t = line.slice(1).trim();
|
|
1191
|
-
if (t.length < BUNDLE_EVIDENCE_MIN_CHARS) continue;
|
|
1192
|
-
if (!byFile.has(cur)) byFile.set(cur, []);
|
|
1193
|
-
byFile.get(cur).push(t);
|
|
1194
|
-
}
|
|
1195
|
-
return byFile;
|
|
1196
|
-
}
|
|
1197
|
-
|
|
1198
|
-
// Pure-ish (fs reads only, injectable): returns { files: [{file, content,
|
|
1199
|
-
// truncated, chars_total}], skipped: [file...], total_chars } for the post-change
|
|
1200
|
-
// contents of changedFiles under cwd. Deleted/unreadable files land in skipped —
|
|
1201
|
-
// their absence from the bundle is visible, never silent.
|
|
1202
|
-
//
|
|
1203
|
-
// Security (panel hacker finding, task 1002660): changedFiles is builder-
|
|
1204
|
-
// controlled and git commits symlinks, so a naive readFileSync would follow a
|
|
1205
|
-
// committed link OUT of the repo (SSH keys, session tokens) and inject the
|
|
1206
|
-
// target into worker prompts + the stored grade. lstat-refuse symlinks and
|
|
1207
|
-
// non-files, and require the resolved path to stay under cwd (containment) —
|
|
1208
|
-
// a refused entry lands in skipped, visible like every other exclusion.
|
|
1209
|
-
function buildContextBundle({ changedFiles, cwd, diff, fsImpl = fs } = {}) {
|
|
1210
|
-
const files = [];
|
|
1211
|
-
const skipped = [];
|
|
1212
|
-
let total = 0;
|
|
1213
|
-
const root = path.resolve(cwd || process.cwd());
|
|
1214
|
-
// task 1002905: the fabricated all-blocker FAIL. A file over the per-file cap
|
|
1215
|
-
// is truncated from the HEAD, so on a large file (ship.js: 250k chars) a change
|
|
1216
|
-
// at char 62k lands entirely outside the bundle — and the prompt presents what
|
|
1217
|
-
// IS bundled as "the post-change contents". A worker then searches the prefix,
|
|
1218
|
-
// doesn't find the new code, and reports it as absent (5 blockers, every claim
|
|
1219
|
-
// exactly the pre-change state). A prefix that contains none of the change is
|
|
1220
|
-
// worse than no bundle at all, so refuse it: skipped[] already tells the worker
|
|
1221
|
-
// to Read the file.
|
|
1222
|
-
const addedByFile = diffAddedLinesByFile(diff);
|
|
1223
|
-
for (const rel of (Array.isArray(changedFiles) ? changedFiles : [])) {
|
|
1224
|
-
if (files.length >= CONTEXT_BUNDLE_MAX_FILES || total >= CONTEXT_BUNDLE_MAX_TOTAL_CHARS) {
|
|
1225
|
-
skipped.push(rel);
|
|
1226
|
-
continue;
|
|
1227
|
-
}
|
|
1228
|
-
if (CONTEXT_BUNDLE_SKIP_RE.test(rel)) { skipped.push(rel); continue; }
|
|
1229
|
-
let content;
|
|
1230
|
-
try {
|
|
1231
|
-
const abs = path.resolve(root, rel);
|
|
1232
|
-
if (abs !== root && !abs.startsWith(root + path.sep)) { skipped.push(rel); continue; }
|
|
1233
|
-
const st = fsImpl.lstatSync(abs);
|
|
1234
|
-
if (!st.isFile() || st.isSymbolicLink()) { skipped.push(rel); continue; }
|
|
1235
|
-
content = fsImpl.readFileSync(abs, 'utf8');
|
|
1236
|
-
} catch (_) { skipped.push(rel); continue; } // deleted by the diff, or unreadable
|
|
1237
|
-
const charsTotal = content.length;
|
|
1238
|
-
let truncated = false;
|
|
1239
|
-
if (content.length > CONTEXT_BUNDLE_MAX_FILE_CHARS) {
|
|
1240
|
-
content = content.slice(0, CONTEXT_BUNDLE_MAX_FILE_CHARS);
|
|
1241
|
-
truncated = true;
|
|
1242
|
-
}
|
|
1243
|
-
if (truncated) {
|
|
1244
|
-
const added = addedByFile.get(rel);
|
|
1245
|
-
if (added && added.length && !added.some((a) => content.includes(a))) { skipped.push(rel); continue; }
|
|
1246
|
-
}
|
|
1247
|
-
if (total + content.length > CONTEXT_BUNDLE_MAX_TOTAL_CHARS) { skipped.push(rel); continue; }
|
|
1248
|
-
total += content.length;
|
|
1249
|
-
files.push({ file: rel, content, truncated, chars_total: charsTotal });
|
|
1250
|
-
}
|
|
1251
|
-
return { files, skipped, total_chars: total };
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
// System 5 (task 1359): an OPT-IN cached path around runSubagent. When the caller
|
|
1255
|
-
// supplies a `cache` spec (from llm-cache.makeCacheSpec), a hit replays the stored
|
|
1256
|
-
// result at cost_usd:0 (so $0 flows naturally into extractCostUsd → session-cost);
|
|
1257
|
-
// a miss runs the untouched runSubagent and stores a clean (exit 0) result. With
|
|
1258
|
-
// no cache spec it's exactly runSubagent — the default path is never altered. The
|
|
1259
|
-
// llm-cache reaches us through the doorway (BV1.R73: it's shared infra core also
|
|
1260
|
-
// uses, so a module can't own it); if it's somehow absent the path degrades to an
|
|
1261
|
-
// ordinary uncached run, exactly as the old lazy-require catch did.
|
|
1262
|
-
async function runSubagentCached({ prompt, opts = {}, cache = null }) {
|
|
1263
|
-
if (!cache || !cache.key) return runSubagent({ prompt, opts });
|
|
1264
|
-
const llmCache = api.llmCache;
|
|
1265
|
-
if (!llmCache) return runSubagent({ prompt, opts });
|
|
1266
|
-
return llmCache.withCache(cache, () => runSubagent({ prompt, opts }));
|
|
1267
|
-
}
|
|
1268
|
-
|
|
1269
|
-
// #486 (ADR 0028 path 1): the `claude -p --output-format json` envelope reports
|
|
1270
|
-
// the subprocess's own dollar cost in `total_cost_usd` (verified present:
|
|
1271
|
-
// {type, ..., total_cost_usd, usage, modelUsage}). We were parsing only
|
|
1272
|
-
// `env.result` (extractAssistantResult) and silently discarding that cost — the
|
|
1273
|
-
// single biggest reason cost_log's `api` category sat at $0.60 across the whole
|
|
1274
|
-
// project (#482 audit F1). Pull it out so the panel's spend can be logged
|
|
1275
|
-
// (category=api, source=grader). Returns null on non-JSON / partial output, so
|
|
1276
|
-
// a cost signal never fabricates a number it doesn't have.
|
|
1277
|
-
function extractCostUsd(rawStdout) {
|
|
1278
|
-
try {
|
|
1279
|
-
const env = JSON.parse(rawStdout);
|
|
1280
|
-
if (env && typeof env.total_cost_usd === 'number' && env.total_cost_usd >= 0) {
|
|
1281
|
-
return env.total_cost_usd;
|
|
1282
|
-
}
|
|
1283
|
-
} catch (_) { /* non-JSON or partial stdout — no cost signal */ }
|
|
1284
|
-
return null;
|
|
1285
|
-
}
|
|
1286
|
-
|
|
1287
|
-
function extractAssistantResult(rawStdout) {
|
|
1288
|
-
try {
|
|
1289
|
-
const env = JSON.parse(rawStdout);
|
|
1290
|
-
if (env && typeof env.result === 'string') return env.result;
|
|
1291
|
-
if (env && typeof env === 'object' && 'functional_fidelity' in env) return env;
|
|
1292
|
-
return rawStdout;
|
|
1293
|
-
} catch (_) {
|
|
1294
|
-
return rawStdout;
|
|
1295
|
-
}
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
// Run one shot end-to-end: build the prompt (caller did this), spawn the
|
|
1299
|
-
// subagent at the requested model, parse + validate. Returns a per-shot
|
|
1300
|
-
// grade-result; subprocess failures become passed=false per-shot results so
|
|
1301
|
-
// aggregateShots can still produce a final verdict.
|
|
1302
|
-
async function runOneShot({ prompt, model, shotIndex, runOpts }) {
|
|
1303
|
-
try {
|
|
1304
|
-
const { stdout, run_ms, exit_code, cost_usd } = await runSubagent({
|
|
1305
|
-
prompt,
|
|
1306
|
-
opts: { ...runOpts, model },
|
|
1307
|
-
});
|
|
1308
|
-
const assistantResult = extractAssistantResult(stdout);
|
|
1309
|
-
const scored = validateAndScore(assistantResult, {
|
|
1310
|
-
run_ms,
|
|
1311
|
-
exit_code,
|
|
1312
|
-
raw_chars: stdout.length,
|
|
1313
|
-
model,
|
|
1314
|
-
shot_index: shotIndex,
|
|
1315
|
-
});
|
|
1316
|
-
// #486 (ADR 0028 path 1): carry this shot's own subprocess cost so the
|
|
1317
|
-
// trivial-diff path (gradeSubagentV3MultiShot → aggregateShots) can sum it
|
|
1318
|
-
// into panel_cost_usd just like the full panel does.
|
|
1319
|
-
scored.cost_usd = cost_usd ?? null;
|
|
1320
|
-
return scored;
|
|
1321
|
-
} catch (err) {
|
|
1322
|
-
return {
|
|
1323
|
-
score: 0,
|
|
1324
|
-
passed: false,
|
|
1325
|
-
threshold: THRESHOLD,
|
|
1326
|
-
grader_kind: 'subagent',
|
|
1327
|
-
grader_version: VERSION,
|
|
1328
|
-
rubric_scores: {},
|
|
1329
|
-
signals: {
|
|
1330
|
-
grader_kind: 'subagent',
|
|
1331
|
-
grader_version: VERSION,
|
|
1332
|
-
model,
|
|
1333
|
-
threshold: THRESHOLD,
|
|
1334
|
-
shot_index: shotIndex,
|
|
1335
|
-
run_ms: err.run_ms ?? null,
|
|
1336
|
-
exit_code: err.exit_code ?? null,
|
|
1337
|
-
subprocess_error: err.code || 'UNKNOWN',
|
|
1338
|
-
stderr_tail: (err.stderr || '').slice(-500),
|
|
1339
|
-
},
|
|
1340
|
-
notes_md: `Shot ${shotIndex}: subprocess failed (${err.code || 'UNKNOWN'}: ${err.message}).`,
|
|
1341
|
-
issues: [],
|
|
1342
|
-
};
|
|
1343
|
-
}
|
|
1344
|
-
}
|
|
1345
|
-
|
|
1346
|
-
// ---------- v4: panel aggregation (MAS Phase 2.0) ----------
|
|
1347
|
-
|
|
1348
|
-
// Collapses N per-worker WorkerResults into the final v4 grade-result.
|
|
1349
|
-
//
|
|
1350
|
-
// Gate logic:
|
|
1351
|
-
// - Any worker `fail` → overall FAIL (a real finding blocks).
|
|
1352
|
-
// - Else any worker errored → overall FALLTHROUGH (manual-confirm) —
|
|
1353
|
-
// a crash is missing signal, not a
|
|
1354
|
-
// finding, so it must not hard-block
|
|
1355
|
-
// on infra flakiness (#126).
|
|
1356
|
-
// - Else any worker `concerns` → overall FALLTHROUGH (manual-confirm).
|
|
1357
|
-
// - All workers `pass` → overall PASS.
|
|
1358
|
-
//
|
|
1359
|
-
// Server-compat forwarding:
|
|
1360
|
-
// The grade POST shape (ff, cq, economy, confidence, issues) is preserved
|
|
1361
|
-
// so the server's existing validateAndScore re-validation aligns with the
|
|
1362
|
-
// panel's verdict. We achieve this by setting the forwarded confidence:
|
|
1363
|
-
// pass → median worker confidence (server's gate also passes)
|
|
1364
|
-
// concerns → FALLTHROUGH_CONFIDENCE (below LOW_CONFIDENCE → server fails)
|
|
1365
|
-
// fail → 0 + a failing axis preserved at < threshold (server fails)
|
|
1366
|
-
//
|
|
1367
|
-
// rubric_scores:
|
|
1368
|
-
// Each axis is the median across workers that reported it (Narc reports
|
|
1369
|
-
// functional_fidelity; Quality reports code_quality + economy). Preserves
|
|
1370
|
-
// the trend dashboard's one-number-per-task contract.
|
|
1371
|
-
//
|
|
1372
|
-
// signals.workers[] carries the full per-worker audit trail.
|
|
1373
91
|
function aggregatePanel(workerResults, meta = {}) {
|
|
1374
92
|
const N = workerResults.length;
|
|
1375
93
|
|