amicus 4.9.1 → 4.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +139 -0
- package/README.md +1 -1
- package/bin/amicus.js +6 -0
- package/docs/configuration.md +52 -0
- package/docs/usage.md +4 -1
- package/electron/main.js +25 -2
- package/electron/setup-ui-alias-groups.js +161 -0
- package/electron/setup-ui-alias-script.js +70 -4
- package/electron/setup-ui-aliases.js +25 -21
- package/electron/setup-ui.js +11 -1
- package/package.json +1 -1
- package/schemas/council-verdict.schema.json +10 -0
- package/src/cli-handlers-doctor.js +9 -16
- package/src/cli-handlers.js +17 -1
- package/src/council/run-retry-window.js +62 -0
- package/src/council/run-retry.js +7 -10
- package/src/council/run-stage2.js +47 -3
- package/src/council/tally.js +12 -0
- package/src/council/verdict-seats-reviewed.js +60 -0
- package/src/council/verdict.js +23 -0
- package/src/headless.js +90 -9
- package/src/utils/api-key-validation.js +183 -94
- package/src/utils/config.js +43 -1
- package/src/utils/degrade.js +8 -0
- package/src/utils/doctor-credit-check.js +61 -0
- package/src/utils/doctor-key-auth-check.js +271 -0
- package/src/utils/live-probes.js +53 -0
- package/src/utils/model-fetcher.js +2 -0
- package/src/utils/model-output-limit.js +124 -0
- package/src/utils/openrouter-credit.js +104 -0
- package/src/utils/session-status.js +73 -0
- package/src/utils/ttft.js +17 -6
|
@@ -5,42 +5,46 @@
|
|
|
5
5
|
* delete, and add functionality for the setup wizard Step 3.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
{ name: 'Other', keys: ['glm', 'minimax', 'grok', 'kimi', 'seed', 'inkling'] },
|
|
17
|
-
];
|
|
8
|
+
const { groupAliases } = require('./setup-ui-alias-groups');
|
|
9
|
+
|
|
10
|
+
/** Attribute/text-safe rendering of user-controlled alias names and routes. */
|
|
11
|
+
function esc(value) {
|
|
12
|
+
return String(value === undefined || value === null ? '' : value)
|
|
13
|
+
.replace(/&/g, '&').replace(/</g, '<')
|
|
14
|
+
.replace(/>/g, '>').replace(/"/g, '"');
|
|
15
|
+
}
|
|
18
16
|
|
|
19
17
|
/**
|
|
20
18
|
* Build the HTML fragment for the alias editor section
|
|
19
|
+
*
|
|
20
|
+
* Issue 213: groups are derived from each alias's ROUTE VENDOR
|
|
21
|
+
* (setup-ui-alias-groups.js), not from a hardcoded list of alias names, so
|
|
22
|
+
* EVERY alias in `aliases` renders exactly once -- the old whitelist silently
|
|
23
|
+
* dropped any name it did not list (12 of 25 in a real config).
|
|
24
|
+
*
|
|
21
25
|
* @param {Object<string,string>} aliases - Map of alias name to model string
|
|
22
26
|
* @returns {string} HTML fragment with search, groups, rows, and add button
|
|
23
27
|
*/
|
|
24
28
|
function buildAliasEditorHTML(aliases) {
|
|
25
29
|
const searchInput = '<input type="text" id="alias-search" class="alias-search" placeholder="Search aliases..." autocomplete="off" spellcheck="false">';
|
|
26
30
|
|
|
27
|
-
const groups =
|
|
31
|
+
const groups = groupAliases(aliases).map(group => {
|
|
28
32
|
const rows = group.keys
|
|
29
|
-
.filter(key => aliases[key] !== undefined)
|
|
30
33
|
.map(key => {
|
|
31
34
|
const model = aliases[key];
|
|
32
|
-
return `<div class="alias-row" data-alias="${key}">` +
|
|
33
|
-
`<span class="alias-name">${key}</span>` +
|
|
35
|
+
return `<div class="alias-row" data-alias="${esc(key)}">` +
|
|
36
|
+
`<span class="alias-name">${esc(key)}</span>` +
|
|
34
37
|
'<span class="alias-arrow">\u2192</span>' +
|
|
35
|
-
`<span class="alias-model">${model}</span>` +
|
|
36
|
-
`<button class="alias-delete" data-alias="${key}">\u00d7</button>` +
|
|
38
|
+
`<span class="alias-model">${esc(model)}</span>` +
|
|
39
|
+
`<button class="alias-delete" data-alias="${esc(key)}">\u00d7</button>` +
|
|
37
40
|
'</div>';
|
|
38
41
|
}).join('\n ');
|
|
39
42
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
// data-vendor records WHICH vendor a group holds, for tests and for anyone
|
|
44
|
+
// inspecting the page. The client does not read it to place rows -- see the
|
|
45
|
+
// SHARED-WITH-THE-BROWSER note in setup-ui-alias-groups.js.
|
|
46
|
+
return `<details class="alias-group" data-vendor="${esc(group.vendor)}">
|
|
47
|
+
<summary>${esc(group.label)} <span class="alias-count">(${group.keys.length})</span></summary>
|
|
44
48
|
${rows}
|
|
45
49
|
</details>`;
|
|
46
50
|
}).join('\n ');
|
|
@@ -82,4 +86,4 @@ function buildAliasEditorHTML(aliases) {
|
|
|
82
86
|
</div>`;
|
|
83
87
|
}
|
|
84
88
|
|
|
85
|
-
module.exports = {
|
|
89
|
+
module.exports = { buildAliasEditorHTML };
|
package/electron/setup-ui.js
CHANGED
|
@@ -22,12 +22,22 @@ const { PROVIDER_FAMILY_NAMES } = require('../src/utils/model-fetcher');
|
|
|
22
22
|
* @param {Object<string,object>} [options.shortlists] - issue 138: per-alias vendor
|
|
23
23
|
* shortlist from buildModelShortlist(), passed through to buildModelStepHTML
|
|
24
24
|
* for the model-level <select>. Defaults to {} (no drill-down rendered).
|
|
25
|
+
* @param {Object<string,string>} [options.aliases] - issue 213: the alias map Step 3
|
|
26
|
+
* renders. Callers pass getEffectiveAliases() (defaults MERGED with the user's
|
|
27
|
+
* config); the default here stays getDefaultAliases() so an omitted option is
|
|
28
|
+
* the old behaviour exactly. This is the second half of issue 213: fixing
|
|
29
|
+
* buildAliasEditorHTML's grouping guarantees "every alias passed in renders
|
|
30
|
+
* exactly once", but the app was only ever passing the 21 built-in defaults,
|
|
31
|
+
* so a user's custom aliases had no row at all. The config that arrives later
|
|
32
|
+
* over IPC cannot repair that -- applyAliasEditsToUI only rewrites the model
|
|
33
|
+
* text of rows that ALREADY exist (`if (!row) { return; }`).
|
|
25
34
|
*/
|
|
26
35
|
function buildSetupHTML(options = {}) {
|
|
27
36
|
const {
|
|
28
37
|
client = 'code-local',
|
|
29
38
|
quickPicks = resolveQuickPicks([]), // pinned fallbacks when not provided
|
|
30
39
|
shortlists = {},
|
|
40
|
+
aliases = getDefaultAliases(), // issue 213
|
|
31
41
|
} = options;
|
|
32
42
|
// Council A1 (PR 215): a pick reaching the page WITHOUT canonicalRoutes makes
|
|
33
43
|
// pickRouteFor fall back to the raw openrouter/... route, which this codebase
|
|
@@ -42,7 +52,7 @@ function buildSetupHTML(options = {}) {
|
|
|
42
52
|
const brandName = getBrandName(client);
|
|
43
53
|
const keysHtml = buildKeysStepHTML(PROVIDERS);
|
|
44
54
|
const modelHtml = buildModelStepHTML(picks, undefined, undefined, shortlists);
|
|
45
|
-
const aliasHtml = buildAliasEditorHTML(
|
|
55
|
+
const aliasHtml = buildAliasEditorHTML(aliases);
|
|
46
56
|
const css = buildWizardCSS();
|
|
47
57
|
const providersJson = JSON.stringify(PROVIDERS);
|
|
48
58
|
const modelChoicesJson = JSON.stringify(picks);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "amicus",
|
|
3
|
-
"version": "4.9.
|
|
3
|
+
"version": "4.9.3",
|
|
4
4
|
"mcpName": "io.github.BourbonDog/amicus",
|
|
5
5
|
"description": "Multi-model LLM Council + parallel AI window for Claude Code. Run structured council reviews across Gemini, GPT, DeepSeek and more — or fork a conversation to any model and fold the results back.",
|
|
6
6
|
"keywords": [
|
|
@@ -229,6 +229,16 @@
|
|
|
229
229
|
"criticSeated"
|
|
230
230
|
]
|
|
231
231
|
},
|
|
232
|
+
"seatsReviewed": {
|
|
233
|
+
"type": "object",
|
|
234
|
+
"description": "#202, optional. How much of the BENCH actually reviewed, derived from runStats: `of` counts every BENCH-role row — `seat`, `critic`, or `lens:<slug>`, exactly the roles seats.js :: buildSeats mints (one per bench seat POST-retry, so a healed seat is counted once and its first attempt is `role:'superseded'`), and `reviewed` counts those whose leg completed. Judges, chair and repair rows are not bench seats and are excluded. EMIT-WHEN-SET: a record with no bench rows carries no key at all, because `0 of 0` would read as a measurement of an empty bench rather than as the absence it is. WHY IT EXISTS: the sibling `seatLoss` above is present only when --critic was requested, and CI runs none — so seat loss was structurally absent from every CI verdict while a two-seat bench published a four-model street-cred table whose dead seats rendered `n/a`, indistinguishable from the legend's neutral (MEASURED, run 4424218c). No `additionalProperties: false` at the top level of this schema means an additive field was always accepted here; this documents the shape rather than changing what is accepted.",
|
|
235
|
+
"properties": {
|
|
236
|
+
"reviewed": { "type": "integer", "minimum": 0, "description": "Bench seats whose leg completed." },
|
|
237
|
+
"of": { "type": "integer", "minimum": 1, "description": "Bench seats benched, post-retry." }
|
|
238
|
+
},
|
|
239
|
+
"required": ["reviewed", "of"],
|
|
240
|
+
"additionalProperties": false
|
|
241
|
+
},
|
|
232
242
|
"degrades": {
|
|
233
243
|
"description": "v4.6 Plan 2: what this run lost — copied verbatim from the sink at verdict assembly. Additive; absent when the sink recorded nothing. v4.9 widened kind with 'info': an announcement that is neither a loss nor a recovery (e.g. a task run's ledger-skipped note) — carried here without degrading the run.",
|
|
234
244
|
"type": "array",
|
|
@@ -17,6 +17,8 @@ const baseUrlCheck = require('./utils/doctor-base-url-check');
|
|
|
17
17
|
// B3 (council review of PR 198, issue 195) — the 'aliases' check body,
|
|
18
18
|
// including its --fix repair of fabricated bare ids. Same split rationale.
|
|
19
19
|
const aliasCheck = require('./utils/doctor-alias-check');
|
|
20
|
+
const creditCheck = require('./utils/doctor-credit-check');
|
|
21
|
+
const keyAuthCheck = require('./utils/doctor-key-auth-check'); // #210 — 'keys' tests presence only; this re-validates.
|
|
20
22
|
|
|
21
23
|
const { DEFAULT_MAX_AGE_MS: MAX_CATALOG_AGE_MS } = require('./utils/model-catalog'); // 24h — single source
|
|
22
24
|
|
|
@@ -32,7 +34,8 @@ function realDeps() {
|
|
|
32
34
|
nodeVersion: process.version,
|
|
33
35
|
readApiKeys: () => require('./utils/api-key-store').readApiKeys(),
|
|
34
36
|
readApiKeyValues: () => require('./utils/api-key-store').readApiKeyValues(),
|
|
35
|
-
checkOpenRouterCredit: (key) =>
|
|
37
|
+
checkOpenRouterCredit: (key) => keyAuthCheck.probeOpenRouterCredit(key), // #210 — same gate as validateApiKey
|
|
38
|
+
validateApiKey: (p, k) => keyAuthCheck.probeApiKey(p, k), // #210
|
|
36
39
|
getCwd: () => process.cwd(),
|
|
37
40
|
readProjectMarkers: (dir) => {
|
|
38
41
|
const exists = (name) => { try { return fs.existsSync(path.join(dir, name)); } catch (_e) { return false; } };
|
|
@@ -141,6 +144,7 @@ async function runDoctorChecks(depsOverride = {}) {
|
|
|
141
144
|
: { id: 'keys', name: 'API keys', status: 'error', message: 'no provider keys configured', hint: 'amicus key <provider> <key> (or run: amicus setup)' };
|
|
142
145
|
}));
|
|
143
146
|
|
|
147
|
+
checks.push(await guardAsync('key-auth', 'API key auth', () => keyAuthCheck.evaluateKeyAuth(d))); // #210
|
|
144
148
|
checks.push((() => {
|
|
145
149
|
try {
|
|
146
150
|
const model = d.resolveModel();
|
|
@@ -211,21 +215,10 @@ async function runDoctorChecks(depsOverride = {}) {
|
|
|
211
215
|
|
|
212
216
|
checks.push(guard('session-metadata-tmp', 'Session metadata tmp files', () => metaSweep.evaluateSessionMetadataTmpSweep(d)));
|
|
213
217
|
|
|
214
|
-
// #43: OpenRouter credit/free-tier — warns (never errors); skipped when no
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
if (!key) {
|
|
219
|
-
return { id: 'openrouter-credit', name: 'OpenRouter credit', status: 'ok', message: 'no OpenRouter key — skipped', hint: null };
|
|
220
|
-
}
|
|
221
|
-
// Reuses the #38 non-blocking probe; resolves warning:null on any failure.
|
|
222
|
-
const res = (await d.checkOpenRouterCredit(key)) || {};
|
|
223
|
-
if (res.warning) {
|
|
224
|
-
return { id: 'openrouter-credit', name: 'OpenRouter credit', status: 'warn', message: res.warning, hint: 'Add credit at openrouter.ai/credits, or build a free council (amicus setup → option 2).' };
|
|
225
|
-
}
|
|
226
|
-
const remaining = (typeof res.limitRemaining === 'number') ? ` ($${res.limitRemaining} remaining)` : '';
|
|
227
|
-
return { id: 'openrouter-credit', name: 'OpenRouter credit', status: 'ok', message: `credit ok${remaining}`, hint: null };
|
|
228
|
-
}));
|
|
218
|
+
// #43: OpenRouter credit/free-tier — warns (never errors); skipped when no
|
|
219
|
+
// key. Body in utils/doctor-credit-check.js (same split as the others).
|
|
220
|
+
checks.push(await guardAsync('openrouter-credit', 'OpenRouter credit',
|
|
221
|
+
() => creditCheck.evaluateOpenRouterCredit(d)));
|
|
229
222
|
|
|
230
223
|
// v4.2 §4.7 C8: configured local / OpenAI-compatible providers (Ollama, LM
|
|
231
224
|
// Studio, vLLM, generic) — reachability only; warn, never error (a napping
|
package/src/cli-handlers.js
CHANGED
|
@@ -176,7 +176,23 @@ async function handleKey(args) {
|
|
|
176
176
|
|
|
177
177
|
console.log(`Validating ${provider} key...`);
|
|
178
178
|
const validation = await validateApiKey(provider, keyArg);
|
|
179
|
-
|
|
179
|
+
// 401 is the only status that means "this credential is not accepted".
|
|
180
|
+
// Everything else — 403 (disabled API, quota, region/bot block), 429, any
|
|
181
|
+
// 5xx, a 404 from a moved endpoint, a Cloudflare 52x during an origin
|
|
182
|
+
// outage, or no status at all because the machine is offline — says
|
|
183
|
+
// something about the REQUEST, not the key. Refusing to save on those is
|
|
184
|
+
// the false ALARM the doctor classifier stopped raising.
|
|
185
|
+
//
|
|
186
|
+
// ⚠️ An ALLOWLIST of what blocks, deliberately. This was a blocklist of
|
|
187
|
+
// what does NOT block, which left every unenumerated status falling through
|
|
188
|
+
// to process.exit(1) while the comment above it claimed only 401 blocked —
|
|
189
|
+
// the code and the narrative disagreed, and the narrative was the nicer of
|
|
190
|
+
// the two. An allowlist cannot rot as new status codes appear.
|
|
191
|
+
const BLOCKS_SAVE = new Set([401]);
|
|
192
|
+
if (!validation.valid && !BLOCKS_SAVE.has(validation.status)) {
|
|
193
|
+
console.warn(`Warning: ${validation.error}`);
|
|
194
|
+
console.warn('Saving the key anyway — run `amicus doctor` to re-check it later.');
|
|
195
|
+
} else if (!validation.valid) {
|
|
180
196
|
console.error(`Error: ${validation.error}`);
|
|
181
197
|
process.exit(1);
|
|
182
198
|
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module council/run-retry-window
|
|
3
|
+
* The Stage-1 retry's no-output window: how long a RELAUNCHED leg may stay
|
|
4
|
+
* silent before the backstop kills it.
|
|
5
|
+
*
|
|
6
|
+
* ⚠️ EXTRACTED, not shaved (release Constraint 6, and the 300-line gate that put
|
|
7
|
+
* `verdict-seat-loss.js` in its own leaf): #219's correction took run-retry.js to
|
|
8
|
+
* 314/300. Its own module also makes the property directly testable.
|
|
9
|
+
*
|
|
10
|
+
* SL-2 Task 5 (#129): a retry re-runs the SAME model under the SAME conditions,
|
|
11
|
+
* so a latency failure is structurally unhealable — double the window rather
|
|
12
|
+
* than repeat it. ⚠️ #135 C0 took the base 240s -> 600s; deliberate, see
|
|
13
|
+
* CHANGELOG (council A1, PR #182).
|
|
14
|
+
*
|
|
15
|
+
* ⚠️ CLAMPED STRICTLY BELOW the leg timeout, not TO it (#219, council gpt
|
|
16
|
+
* major). `Math.min(2 * backstop, legTimeoutMs)` made the two deadlines EQUAL
|
|
17
|
+
* whenever `2 * backstop >= legTimeoutMs` — exactly CI today (2 x 480000 ===
|
|
18
|
+
* 960000 === `--timeout 16`). The backstop still won, but only by epsilon and
|
|
19
|
+
* only because the poll loop tests its deadline BEFORE sleeping, so the final
|
|
20
|
+
* poll lands just past the wall. That is an undocumented accident of loop order;
|
|
21
|
+
* if it ever lost, the leg would die a generic `timeout` and throw away the
|
|
22
|
+
* named NO_OUTPUT_BACKSTOP diagnosis this clamp exists to preserve.
|
|
23
|
+
*
|
|
24
|
+
* A PROPORTIONAL headroom, not a fixed subtraction: a constant large enough to
|
|
25
|
+
* beat a poll cycle (seconds) would drive a small leg cap to zero or negative,
|
|
26
|
+
* and `ms <= 0` is the documented DISABLE hatch — silently disabling the backstop
|
|
27
|
+
* is far worse than the race it fixes. 5% of any realistic leg cap clears the 2 s
|
|
28
|
+
* poll interval by a wide margin.
|
|
29
|
+
*
|
|
30
|
+
* `2 * 0 === 0` still disables, because `Math.min(0, anything positive) === 0`.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
'use strict';
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @param {number} baseBackstopMs the first attempt's resolved no-output window
|
|
37
|
+
* @param {number} legTimeoutMs the per-leg hard cap ((o.timeout || 15) * 60_000)
|
|
38
|
+
* @returns {number} the retry's window: doubled, clamped strictly below the cap
|
|
39
|
+
*/
|
|
40
|
+
function retryBackstopMs(baseBackstopMs, legTimeoutMs) {
|
|
41
|
+
// ⚠️ NOT floored at the first attempt's window, and #219 round 2 (glm) asked
|
|
42
|
+
// for exactly that — correctly observing that when `legTimeoutMs <= 2 * base`
|
|
43
|
+
// the retry window comes out slightly SHORTER than the attempt it exists to
|
|
44
|
+
// give room to (480000/480000 -> 456000). The observation is right; the remedy
|
|
45
|
+
// is worse than what it fixes, MEASURED across all three regimes:
|
|
46
|
+
//
|
|
47
|
+
// regime first(effective) unfloored floored unfloored gives
|
|
48
|
+
// cap = 2x base 480000 912000 912000 NAMED backstop
|
|
49
|
+
// cap = base 480000 456000 480000 NAMED backstop
|
|
50
|
+
// cap < base (t=3) 180000 171000 180000 NAMED backstop
|
|
51
|
+
//
|
|
52
|
+
// Flooring pins the window ONTO the leg cap in both degenerate regimes, which
|
|
53
|
+
// is the tie the headroom exists to break — so the leg dies a generic
|
|
54
|
+
// `timeout` and the named diagnosis is lost. That diagnosis is this module's
|
|
55
|
+
// entire purpose. The unfloored cost is bounded at 5% of the window (24 s at
|
|
56
|
+
// CI scale, 9 s at `--timeout 3`), and it is paid only where the leg cap
|
|
57
|
+
// already dominates the backstop. Trading ≤5% of one retry's patience for a
|
|
58
|
+
// named cause on every retry death is the right side of that trade.
|
|
59
|
+
return Math.min(2 * baseBackstopMs, Math.floor(legTimeoutMs * 0.95));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
module.exports = { retryBackstopMs };
|
package/src/council/run-retry.js
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
const { materializeReviews, isAbortExit } = require('./run-launch');
|
|
18
18
|
const runState = require('./run-state');
|
|
19
19
|
const { resolveNoOutputBackstopMs } = require('../utils/no-output-backstop');
|
|
20
|
+
const { retryBackstopMs } = require('./run-retry-window');
|
|
20
21
|
const { waveStillDeadNote, srcLegStillDeadNote, retryLegStillDeadNote, missingLegStillDeadNote }
|
|
21
22
|
= require('./run-retry-notes');
|
|
22
23
|
// briefingFor + bindRetryWave live in ./run-retry-launch (v4.8 T-A2 split); the pad/bind core it wraps is stage1-bind.js :: bindPaddedWave (SI-27).
|
|
@@ -54,17 +55,13 @@ async function retryStage1Losses(ctx, { deadWaves = [], deadLegs = [],
|
|
|
54
55
|
const out = { aborted: null, recoveredLegs: [], stillDeadNotes: [], twins,
|
|
55
56
|
stillDeadWaves: [], stillDeadLegs: [], skippedDeadWaves: [], skippedDeadLegs: [],
|
|
56
57
|
stillDeadRetryLegs: [], seatOf: new Map(), orphanLegs: [], attemptedSeats: new Set() };
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
// the
|
|
60
|
-
// silently becoming an ordinary timeout at a low --timeout. 2*0 === 0 keeps
|
|
61
|
-
// the disable hatch. (o.timeout || 15) * 60 * 1000 mirrors fanout.js:254.
|
|
62
|
-
// ⚠️ #135 C0 took this 240s -> 600s; deliberate, see CHANGELOG (council A1, PR #182).
|
|
58
|
+
// The retry window: doubled and clamped strictly below the leg timeout.
|
|
59
|
+
// Reasoning (and #219's correction) lives in ./run-retry-window — extracted
|
|
60
|
+
// for the 300-line gate. (o.timeout || 15) * 60 * 1000 mirrors fanout.js:254.
|
|
63
61
|
const legTimeoutMs = (o.timeout || 15) * 60 * 1000;
|
|
64
|
-
const escalatedBackstopMs =
|
|
65
|
-
|
|
66
|
-
legTimeoutMs
|
|
67
|
-
);
|
|
62
|
+
const escalatedBackstopMs = retryBackstopMs(
|
|
63
|
+
Number.isFinite(o.noOutputBackstopMs) ? o.noOutputBackstopMs : resolveNoOutputBackstopMs(),
|
|
64
|
+
legTimeoutMs);
|
|
68
65
|
|
|
69
66
|
for (const unit of groupStage1Losses(o, deadWaves, deadLegs, seatOf, twins)) {
|
|
70
67
|
// Task-4 review hardening: a unit this pass cannot even ATTEMPT — an
|
|
@@ -23,6 +23,9 @@ const stage2 = require('./briefings-stage2');
|
|
|
23
23
|
const { parseJudgeOutput } = require('./parse-stage2');
|
|
24
24
|
const { sanitizeName, isAbortExit } = require('./run-launch');
|
|
25
25
|
const runState = require('./run-state');
|
|
26
|
+
// #219 (council, glm minor): `leg.error` is UNTRUSTED provider text. The house
|
|
27
|
+
// sanitizer — one sanitizer, one dialect (utils/text-sanitize.js).
|
|
28
|
+
const { collapseExcerpt } = require('../utils/text-sanitize');
|
|
26
29
|
const { buildRunStatsEntry } = require('./run-assemble');
|
|
27
30
|
// v4.8 PR3 Task 4: seat binding. artifactName is NOT re-exported from
|
|
28
31
|
// run-launch.js (its exports stop at sanitizeName/isAbortExit), so it comes
|
|
@@ -169,9 +172,16 @@ async function runStage2(ctx, { reviews, labels, globalFindings, extraLabeled =
|
|
|
169
172
|
fs.writeFileSync(path.join(o.runDir, name), leg.summary, { mode: 0o600 });
|
|
170
173
|
}
|
|
171
174
|
let conformance = 'clean';
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
+
// #202: ONE predicate for "this judge never answered at all", shared by the
|
|
176
|
+
// DEAD_LEG classification below and by the degrade it now raises. Spelling it
|
|
177
|
+
// twice is how the two would drift into disagreeing about which judges died —
|
|
178
|
+
// and note it is NOT `leg.status !== 'complete'`: a leg that completes with an
|
|
179
|
+
// EMPTY summary produced nothing either, and the DEAD_LEG arm has always
|
|
180
|
+
// treated it that way.
|
|
181
|
+
const legDied = !(leg.status === 'complete' && leg.summary);
|
|
182
|
+
let parsed = legDied
|
|
183
|
+
? { ok: false, errors: [{ code: 'DEAD_LEG', detail: leg.error || leg.status }] }
|
|
184
|
+
: parseJudgeOutput(leg.summary, parseCtx);
|
|
175
185
|
let attempts = 0;
|
|
176
186
|
// ⚠️ LC-12: the judging text the repair prompt must carry, tracked exactly like
|
|
177
187
|
// Stage-1's `repairing` so `judging` and `parsed.errors` always describe the SAME
|
|
@@ -216,6 +226,40 @@ async function runStage2(ctx, { reviews, labels, globalFindings, extraLabeled =
|
|
|
216
226
|
if (parsed.ok) { conformance = 'repaired'; }
|
|
217
227
|
}
|
|
218
228
|
if (!parsed.ok) {
|
|
229
|
+
// #202: THE MISSING THIRD CASE. A dead judge leg still comes back as a leg
|
|
230
|
+
// object, so bindPaddedWave binds it — it is neither `orphan` nor
|
|
231
|
+
// `unbound`, and stage 2 had no case for it. It fell through into
|
|
232
|
+
// judgeResults with `ok:false` and vanished: MEASURED on CI run
|
|
233
|
+
// 32956900910 (wave 9d8029c8-s2), where glm and qwen judges died at +300s
|
|
234
|
+
// with zero tokens and run.json recorded no degrade at all, while the
|
|
235
|
+
// verdict shipped a four-column adjudication matrix two of them never
|
|
236
|
+
// voted in. The one net that might have caught it, `thin-cross-review`,
|
|
237
|
+
// fires only at `usableJudges < 2`; that run had exactly 2 of 4.
|
|
238
|
+
//
|
|
239
|
+
// ⚠️ Emitted with the default kind ('degrade'), so run-degrade.js's sink
|
|
240
|
+
// sets `degraded.value` and the run exits 2. That is a deliberate
|
|
241
|
+
// behaviour change (owner's call): before it, a half-adjudicated verdict
|
|
242
|
+
// could exit 0, and W11 only exited 2 because of an unrelated
|
|
243
|
+
// cost-accounting degrade. An unparseable-but-ANSWERED judge is a
|
|
244
|
+
// different fact and is deliberately excluded — it already darkens the
|
|
245
|
+
// seat's row via `conformance: 'unstructured'`, and it is repairable.
|
|
246
|
+
if (legDied) {
|
|
247
|
+
ctx.degrade.note({
|
|
248
|
+
channel: 'stage2-judge',
|
|
249
|
+
what: `judge ${judge} did not adjudicate`,
|
|
250
|
+
// #219: `why` is PROSE — it renders into run.json, the report and the
|
|
251
|
+
// sticky PR comment — so the provider's text is collapsed to one
|
|
252
|
+
// bounded line. `data.reason` below stays VERBATIM on purpose: it is
|
|
253
|
+
// the machine surface, it is JSON (nothing to inject), and truncating
|
|
254
|
+
// it would cost exactly the fidelity a reader opens run.json for.
|
|
255
|
+
why: `its Stage-2 leg ended '${leg.status}'`
|
|
256
|
+
+ (leg.error ? `: ${collapseExcerpt(leg.error, 200)}` : ''),
|
|
257
|
+
effect: `the cross-review was adjudicated by fewer than the ${judges.length} judges the `
|
|
258
|
+
+ 'bench implies; the run continues and will exit degraded (2)',
|
|
259
|
+
data: { judge, seat: seat ? seat.id : null, waveId: `${o.runId}-s2`,
|
|
260
|
+
status: leg.status, reason: leg.error || null },
|
|
261
|
+
});
|
|
262
|
+
}
|
|
219
263
|
judgeResults.push({ judge, seat, ok: false, order: null, orderSeats: null, adjudications: null,
|
|
220
264
|
conformance: leg.status === 'complete' ? 'unstructured' : 'clean',
|
|
221
265
|
// #83 (v4.6 Plan 2): the judge's ORIGINAL Stage-2 wave leg, mirroring
|
package/src/council/tally.js
CHANGED
|
@@ -6,6 +6,11 @@ const { peersOf, unattributedPeerDrops } = require('./peer-split');
|
|
|
6
6
|
// the seat-keying in it — release Constraint 6 is EXTRACT, never shave).
|
|
7
7
|
// computeStreetCred is re-exported below, so no existing import path moved.
|
|
8
8
|
const { computeStreetCred } = require('./street-cred');
|
|
9
|
+
// #202: the TTFT probe's LAST emit gate — a RE-PROJECTION, so omitting the field
|
|
10
|
+
// here destroyed one already produced rather than failing to produce it. Through
|
|
11
|
+
// v4.9.1 utils/ttft.js's docblock enumerated only the four PRODUCER gates and
|
|
12
|
+
// stopped one short of this one; it now names all five.
|
|
13
|
+
const { isMeasuredTtft } = require('../utils/ttft');
|
|
9
14
|
|
|
10
15
|
/**
|
|
11
16
|
* Peers-only tier cascade. a/d are agree/dispute counts among PEER judges
|
|
@@ -181,6 +186,13 @@ function tally(input) {
|
|
|
181
186
|
...(r.seat ? { seat: r.seat } : {}),
|
|
182
187
|
status: r.status || 'unknown',
|
|
183
188
|
durationMs: typeof r.durationMs === 'number' ? r.durationMs : null,
|
|
189
|
+
// #202: emit-when-VALID, in buildRunStatsEntry's own slot (between
|
|
190
|
+
// durationMs and usage) so G7b's key-order invariant holds for a row that
|
|
191
|
+
// carries it. NOT `durationMs`'s null-coercion above: a null here would be
|
|
192
|
+
// read as a measurement, and absence must keep its one meaning — "no
|
|
193
|
+
// substantive tick was ever observed". The shared predicate is imported
|
|
194
|
+
// rather than hand-spelled; this file has no require-free pin.
|
|
195
|
+
...(isMeasuredTtft(r.ttftMs) ? { ttftMs: r.ttftMs } : {}),
|
|
184
196
|
usage: r.usage || null,
|
|
185
197
|
})),
|
|
186
198
|
tierCounts: countTiers(outFindings),
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module council/verdict-seats-reviewed
|
|
3
|
+
* #202: the bench-seat census for verdict.json, as a spreadable fragment.
|
|
4
|
+
*
|
|
5
|
+
* ⚠️ EXTRACTED, not shaved — release Constraint 6, and the same 300-line gate
|
|
6
|
+
* that put `verdict-seat-loss.js` in its own leaf: adding this to verdict.js
|
|
7
|
+
* took that file to 313/300. It could not join verdict-seat-loss.js either —
|
|
8
|
+
* that module is pinned to export EXACTLY its two functions.
|
|
9
|
+
*
|
|
10
|
+
* This is the ONE place that decides what "a bench seat" means, so the
|
|
11
|
+
* emit-when-set rule and the role filter cannot drift apart. `of` is every
|
|
12
|
+
* `role:'seat'` row — one per bench seat POST-retry, so a healed seat counts
|
|
13
|
+
* once while its first attempt is `role:'superseded'`; judges, chair and
|
|
14
|
+
* repairs are not bench seats. `reviewed` is those whose leg completed: a
|
|
15
|
+
* `timeout` is not a review any more than an `error` is.
|
|
16
|
+
*
|
|
17
|
+
* A LEAF: it requires nothing, matching its seat-loss sibling.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
'use strict';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @param {Array<object>|undefined} runStats
|
|
24
|
+
* @returns {{seatsReviewed?: {reviewed: number, of: number}}}
|
|
25
|
+
*/
|
|
26
|
+
/**
|
|
27
|
+
* Is this runStats row a BENCH seat — something that was asked to review?
|
|
28
|
+
*
|
|
29
|
+
* ⚠️ These are exactly the three roles `seats.js :: buildSeats` mints, and that
|
|
30
|
+
* is the point: it is the producer, so this mirrors it rather than guessing.
|
|
31
|
+
* `role === 'seat'` alone (#219) counted ZERO on a `--lenses` run, where every
|
|
32
|
+
* seat carries `lens:<slug>` — so emit-when-set silently omitted the census from
|
|
33
|
+
* the runs using the richest bench. A critic counts too: it is an adversarial
|
|
34
|
+
* seat, but it reviews.
|
|
35
|
+
*
|
|
36
|
+
* An ALLOWLIST, not a denylist of judge/chair/repair/superseded: a new
|
|
37
|
+
* non-bench role added later must not silently inflate the denominator.
|
|
38
|
+
*/
|
|
39
|
+
function isBenchRole(role) {
|
|
40
|
+
return role === 'seat' || role === 'critic'
|
|
41
|
+
|| (typeof role === 'string' && role.startsWith('lens:'));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function seatsReviewedOf(runStats) {
|
|
45
|
+
// ⚠️ `Array.isArray`, NOT `runStats || []`. buildVerdict is reachable on
|
|
46
|
+
// externally-supplied records that never touched tally() in-process — the MCP
|
|
47
|
+
// `record` param of mcp-tools.js :: amicus_verdict is `z.record(z.any())`,
|
|
48
|
+
// fully permissive — and this file's own tests hand it `runStats: {}`. A
|
|
49
|
+
// truthy non-array sails past `||` and throws on `.filter`, turning a missing
|
|
50
|
+
// census into a crashed verdict build. The closed-literal comment further down
|
|
51
|
+
// makes the same argument about the same caller.
|
|
52
|
+
const seats = (Array.isArray(runStats) ? runStats : []).filter(r => r && isBenchRole(r.role));
|
|
53
|
+
if (seats.length === 0) { return {}; }
|
|
54
|
+
return { seatsReviewed: {
|
|
55
|
+
reviewed: seats.filter(r => r.status === 'complete').length,
|
|
56
|
+
of: seats.length,
|
|
57
|
+
} };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
module.exports = { seatsReviewedOf };
|
package/src/council/verdict.js
CHANGED
|
@@ -13,6 +13,10 @@ const { summarizeSeatLoss, deriveSeatLoss } = require('./verdict-seat-loss');
|
|
|
13
13
|
// opts.overallVerdict, null in every Stage-4 manual path).
|
|
14
14
|
const VERDICT_SCHEMA_VERSION = 2;
|
|
15
15
|
|
|
16
|
+
// #202: the bench-seat census leaf (the 300-line gate — same reason
|
|
17
|
+
// verdict-seat-loss.js is its own module).
|
|
18
|
+
const { seatsReviewedOf } = require('./verdict-seats-reviewed');
|
|
19
|
+
|
|
16
20
|
/**
|
|
17
21
|
* Merge a tally record with Claude's Stage-4 decisions into the verdict record.
|
|
18
22
|
* @param {object} record tally() output
|
|
@@ -123,6 +127,25 @@ function buildVerdict(record, decisions = [], opts = {}) {
|
|
|
123
127
|
// Additive and OPTIONAL (schemaVersion stays 2): present only when a critic
|
|
124
128
|
// was requested, so its absence never has to be interpreted.
|
|
125
129
|
...(opts.seatLoss ? { seatLoss: opts.seatLoss } : {}),
|
|
130
|
+
// #202: how much of the bench actually reviewed. DERIVED here rather than
|
|
131
|
+
// passed in, because every caller that could pass it already has the same
|
|
132
|
+
// `runStats` this reads — and a parameter is one more thing a rebuild path
|
|
133
|
+
// can forget (the `intent` key needed a SECOND carrier for exactly that).
|
|
134
|
+
//
|
|
135
|
+
// Its sibling `seatLoss` cannot serve: `deriveSeatLoss` returns null when no
|
|
136
|
+
// `--critic` was requested, and CI runs `CRITIC: ''` — so seat loss is
|
|
137
|
+
// STRUCTURALLY absent from every CI verdict. MEASURED on run 4424218c, a
|
|
138
|
+
// two-seat bench that published a four-model street-cred table with the dead
|
|
139
|
+
// seats rendered `n/a`, indistinguishable from the legend's "neutral".
|
|
140
|
+
//
|
|
141
|
+
// Counts the BENCH roles buildSeats mints — `seat`, `critic` and `lens:<slug>`
|
|
142
|
+
// (#219 r2: this said "`role:'seat'` ONLY" after the filter was widened, and
|
|
143
|
+
// two seats caught the stale sentence). One row per bench seat POST-retry, so a
|
|
144
|
+
// healed seat is counted once (its first attempt is `role:'superseded'`), and
|
|
145
|
+
// judges/chair/repairs are not bench seats. Emit-when-set — a record with no
|
|
146
|
+
// bench rows carries no key, because `0 of 0` would read as a measurement of
|
|
147
|
+
// an empty bench rather than as the absence it is.
|
|
148
|
+
...seatsReviewedOf(record.runStats),
|
|
126
149
|
// v4.6 Plan 2 (spec §4): the canonical what-was-lost surface. Additive and
|
|
127
150
|
// OPTIONAL — present only when the run actually degraded, so a clean run's
|
|
128
151
|
// verdict is byte-for-byte unchanged. schemaVersion stays 2 (the v4.5.2
|