amicus 4.6.2 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +349 -0
- package/README.md +24 -13
- package/bin/amicus.js +31 -0
- package/docs/ROADMAP.md +172 -36
- package/docs/configuration.md +56 -6
- package/docs/council.md +63 -10
- package/docs/doc-system.md +8 -7
- package/docs/schemas.md +10 -1
- package/docs/troubleshooting.md +27 -1
- package/docs/usage.md +68 -15
- package/electron/workspace-ui/index.html +3 -0
- package/electron/workspace-ui/live-model.js +132 -21
- package/electron/workspace-ui/workspace-app.js +20 -4
- package/electron/workspace-ui/workspace-lazy.js +233 -0
- package/electron/workspace-ui/workspace-matrix.js +12 -1
- package/electron/workspace-ui/workspace-panels.js +24 -171
- package/electron/workspace-ui/workspace-render.js +15 -5
- package/electron/workspace-ui/workspace-seats.js +88 -5
- package/electron/workspace-ui/workspace-verbs.js +1 -1
- package/electron/workspace-ui/workspace.css +6 -0
- package/package.json +5 -2
- package/schemas/council-run.schema.json +1 -0
- package/schemas/council-stats.schema.json +9 -1
- package/schemas/run.schema.json +2 -1
- package/schemas/spend.schema.json +1 -1
- package/schemas/wave.schema.json +2 -1
- package/skills/second-opinion/MANUAL-ORCHESTRATION.md +12 -0
- package/skills/second-opinion/MODEL-NOTES.md +5 -4
- package/skills/sidecar/SKILL.md +7 -2
- package/src/cli-council-run-bench.js +86 -0
- package/src/cli-handlers-council-run.js +65 -81
- package/src/cli-handlers-council.js +24 -3
- package/src/cli-handlers-doctor.js +9 -3
- package/src/cli-handlers-fanout.js +179 -0
- package/src/cli-handlers-pack.js +24 -10
- package/src/cli-handlers-run.js +19 -161
- package/src/cli-template-args.js +48 -0
- package/src/cli.js +39 -46
- package/src/council/debate.js +89 -10
- package/src/council/ledger.js +72 -11
- package/src/council/presets-cli.js +6 -2
- package/src/council/report.js +17 -6
- package/src/council/run-assemble.js +15 -3
- package/src/council/run-budget.js +2 -2
- package/src/council/run-chair.js +70 -11
- package/src/council/run-debate.js +51 -67
- package/src/council/run-launch.js +9 -2
- package/src/council/run-retry.js +4 -1
- package/src/council/run-stage1-launch.js +94 -0
- package/src/council/run-stage2.js +25 -4
- package/src/council/run-stages.js +79 -86
- package/src/council/run-state.js +10 -2
- package/src/council/run.js +26 -2
- package/src/council/tally.js +6 -2
- package/src/mcp-council-awareness.js +1 -0
- package/src/mcp-council-bench.js +4 -0
- package/src/mcp-council-run.js +10 -0
- package/src/mcp-server.js +114 -54
- package/src/mcp-tools.js +12 -5
- package/src/pack/pack-cli.js +1 -1
- package/src/pack/pack-forward.js +12 -4
- package/src/pack/pack-resolve.js +3 -0
- package/src/pack/pack-store.js +20 -3
- package/src/pack/pack-validate.js +5 -1
- package/src/session-manager.js +6 -2
- package/src/sidecar/budget.js +38 -4
- package/src/sidecar/fanout-budget.js +1 -2
- package/src/sidecar/fanout-leg-fallback.js +7 -3
- package/src/sidecar/fanout-wave-io.js +13 -1
- package/src/sidecar/fanout.js +11 -9
- package/src/sidecar/list-limit.js +50 -0
- package/src/sidecar/list-search.js +69 -0
- package/src/sidecar/read.js +90 -5
- package/src/sidecar/start-metadata.js +58 -0
- package/src/sidecar/start.js +8 -43
- package/src/sidecar/workspace-auto-open.js +2 -2
- package/src/spend-query.js +2 -1
- package/src/template/apply.js +7 -4
- package/src/template/render.js +6 -2
- package/src/template/store.js +1 -1
- package/src/utils/alias-audit.js +19 -0
- package/src/utils/cli-preflight.js +27 -1
- package/src/utils/config.js +15 -0
- package/src/utils/curated-models.js +43 -7
- package/src/utils/gateway-route-audit.js +16 -3
- package/src/utils/model-fetcher.js +8 -6
- package/src/utils/remediation-hints.js +14 -0
- package/src/utils/result-schema-rebuild.js +1 -0
- package/src/utils/result-schema.js +6 -1
- package/src/utils/session-index-tmp-sweep.js +18 -3
- package/src/utils/session-index.js +1 -0
- package/src/utils/session-metadata-tmp-sweep.js +156 -0
- package/src/utils/spend-ledger.js +11 -4
- package/src/utils/validators.js +16 -0
package/src/cli.js
CHANGED
|
@@ -105,8 +105,12 @@ function parseArgs(argv) {
|
|
|
105
105
|
// Single short-flag alias, scoped to exactly '-o' (council verdict's
|
|
106
106
|
// --out shorthand). No general short-flag support is implemented —
|
|
107
107
|
// any other leading-dash token still falls through to positionals.
|
|
108
|
+
// R5: deliberately lockstep with the long-option branch above
|
|
109
|
+
// (`!next.startsWith('--')`) — '-o' and '--out' must consume values
|
|
110
|
+
// identically. A dash-leading value is still accepted here (parsing
|
|
111
|
+
// only normalizes); cli-handlers-council.js's R1 guard rejects it.
|
|
108
112
|
const next = argv[i + 1];
|
|
109
|
-
if (next && !next.startsWith('
|
|
113
|
+
if (next && !next.startsWith('--')) {
|
|
110
114
|
result.out = next;
|
|
111
115
|
i++;
|
|
112
116
|
} else {
|
|
@@ -455,6 +459,7 @@ Options for 'start':
|
|
|
455
459
|
--artifact <file> File whose content fills {{artifact}} (256 KB cap; needs --template)
|
|
456
460
|
--var <k=v> Template variable, repeatable (needs --template)
|
|
457
461
|
--pack <name|path> Load a saved pack (model/options/template); explicit flags override it
|
|
462
|
+
--tag <t> Label this session for list/search/spend grouping (1-64 chars, [A-Za-z0-9_-])
|
|
458
463
|
`,
|
|
459
464
|
fanout: `
|
|
460
465
|
Options for 'fanout':
|
|
@@ -465,31 +470,34 @@ Options for 'fanout':
|
|
|
465
470
|
~32KB Windows argument cap). Mutually exclusive
|
|
466
471
|
with --prompt. Also works with 'start'.
|
|
467
472
|
--wave-id <id> Explicit wave ID (leg IDs become <id>-1..N)
|
|
468
|
-
--retry-failed <waveId>
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
+
--retry-failed <waveId> Relaunch ONLY that wave's failed/timed-out/crashed/
|
|
474
|
+
aborted legs as a NEW linked wave, using each leg's
|
|
475
|
+
own saved context (byte-identical retry). Skips
|
|
476
|
+
--prompt/--models; --models filters which failed
|
|
477
|
+
legs to retry. wave.json is never modified.
|
|
473
478
|
--json Emit the wave result as stable JSON on stdout
|
|
479
|
+
--quiet Suppress per-leg progress ticks, the wave-result summary, and
|
|
480
|
+
preflight errors
|
|
474
481
|
--max-cost <$> Refuse the wave if the estimated total exceeds $ (soft ceiling)
|
|
475
482
|
--no-cost-gate Disable the budget gate (per-$/Mtok threshold + ceiling) for this run
|
|
476
483
|
--fallback / --no-fallback Opt-in cheaper-model substitution on a classified
|
|
477
484
|
rate-limit/overload leg failure (spec 6.2). Overrides
|
|
478
485
|
config fallbacks.enabled when passed; default: config, else off.
|
|
479
|
-
--gateway <mode>
|
|
480
|
-
--follow
|
|
481
|
-
--on-complete <cmd>
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
486
|
+
--gateway <mode> Routing: auto (direct-first), direct, or openrouter
|
|
487
|
+
--follow Stream this run's events to stderr as they happen (--json -> NDJSON)
|
|
488
|
+
--on-complete <cmd> Run a shell command once, at terminal state, after
|
|
489
|
+
wave.json is durable. The command is user-authored
|
|
490
|
+
on THIS command line (CLI-only — never sourced from
|
|
491
|
+
config/briefings/model output); payload rides via
|
|
492
|
+
env only (AMICUS_TASK_ID/TYPE/STATUS/EXIT_CODE/
|
|
493
|
+
RESULT_FILE/EVENTS_FILE/COST/PROJECT), never model
|
|
494
|
+
text. Child stdout/stderr go to amicus stderr.
|
|
495
|
+
Never changes the wave's exit code, docs, or events.
|
|
489
496
|
--template <name|path> Render a briefing template ({{prompt}}, {{artifact}}, {{var.*}})
|
|
490
497
|
--artifact <file> File whose content fills {{artifact}} (256 KB cap; needs --template)
|
|
491
498
|
--var <k=v> Template variable, repeatable (needs --template)
|
|
492
499
|
--pack <name|path> Load a saved pack (bench/options/template); explicit flags override it
|
|
500
|
+
--tag <t> Label this wave for list/search/spend grouping (1-64 chars, [A-Za-z0-9_-])
|
|
493
501
|
Shared per-leg knobs: --agent, --thinking, --timeout, --summary-length,
|
|
494
502
|
--no-context, --context-*, --mcp*, --no-validate-model, --cwd
|
|
495
503
|
Exit codes: 0 all legs complete, 2 partial, 1 none complete / hard failure
|
|
@@ -511,6 +519,8 @@ Options for 'models':
|
|
|
511
519
|
Options for 'list':
|
|
512
520
|
--status <filter> Filter by status (running, complete)
|
|
513
521
|
--all Show all projects
|
|
522
|
+
--search <q> Case-insensitive substring filter over id, tag, and briefing
|
|
523
|
+
--limit <n> Show only the n newest rows (0 = unlimited)
|
|
514
524
|
--json Output as JSON
|
|
515
525
|
`,
|
|
516
526
|
status: `
|
|
@@ -580,7 +590,7 @@ Subcommands for 'council':
|
|
|
580
590
|
[--debate] [--claude-review <file>] [--no-cost-gate] [--follow]
|
|
581
591
|
[--fallback] [--no-fallback] [--on-complete <cmd>]
|
|
582
592
|
[--template <name|path>] [--artifact <file>] [--var <k=v>]
|
|
583
|
-
[--pack <name|path>]
|
|
593
|
+
[--pack <name|path>] [--tag <t>]
|
|
584
594
|
Run the full headless council engine (v4.0).
|
|
585
595
|
Chair default: deepseek (must NOT be a bench seat).
|
|
586
596
|
--critic and --lenses are mutually exclusive.
|
|
@@ -633,7 +643,7 @@ Options for 'spend':
|
|
|
633
643
|
--model <id-or-prefix> Only rows whose model starts with this
|
|
634
644
|
--op <start|continue|resume|leg> Only rows with this operation
|
|
635
645
|
--failed Only non-complete (wasted) rows
|
|
636
|
-
--group-by <model|wave|council|project|op|day> Rollup dimension (default model)
|
|
646
|
+
--group-by <model|wave|council|project|op|day|tag> Rollup dimension (default model)
|
|
637
647
|
--rows Include matching raw rows (capped at 1000)
|
|
638
648
|
--json Machine-readable output (versioned spend doc)
|
|
639
649
|
Reads ~/.config/amicus/spend-ledger.jsonl (one row per completed run/leg).
|
|
@@ -691,34 +701,17 @@ Options for 'template':
|
|
|
691
701
|
pack: `
|
|
692
702
|
Options for 'pack':
|
|
693
703
|
amicus pack save <name> --kind council|fanout|solo [flags]
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
--template <name|path> briefing template
|
|
706
|
-
reference (not rendered)
|
|
707
|
-
--version <semver> default 1.0.0 (an
|
|
708
|
-
unchanged re-save is a
|
|
709
|
-
no-op; a changed one
|
|
710
|
-
auto-bumps the patch)
|
|
711
|
-
--description <text>
|
|
712
|
-
amicus pack save <name> --from-run <id>
|
|
713
|
-
Build a pack from an existing council run /
|
|
714
|
-
fanout wave / solo session instead of flags
|
|
715
|
-
(models, options, and a template REFERENCE only
|
|
716
|
-
— briefing text is never captured)
|
|
717
|
-
amicus pack list [--json] List saved packs
|
|
718
|
-
amicus pack show <name|path> [--json]
|
|
719
|
-
Print a pack plus its validation report (never
|
|
720
|
-
fails on an invalid pack — see 'validation')
|
|
721
|
-
amicus pack rm <name> [--json] Remove a saved pack
|
|
704
|
+
amicus pack save <name> --from-run <id> Build from an existing run (models/
|
|
705
|
+
options/template reference only — briefing never captured)
|
|
706
|
+
amicus pack list [--json] List saved packs
|
|
707
|
+
amicus pack show <name|path> [--json] Print a pack + validation report
|
|
708
|
+
amicus pack rm <name> [--json] Remove a saved pack
|
|
709
|
+
Flags: --bench <a,b,c|name> (council/fanout) · --model (solo) · --chair/--critic/--lenses
|
|
710
|
+
(council) · --debate/--no-debate (council) · --timeout/--max-cost/--gateway (shared) ·
|
|
711
|
+
--agent/--thinking/--summary-length (fanout/solo) · --template <name|path> (reference only,
|
|
712
|
+
not rendered) · --pack-version <semver> (the SAVED PACK's version, default 1.0.0; unchanged
|
|
713
|
+
re-save is a no-op, changed auto-bumps patch — NOT --version, which is amicus's own global
|
|
714
|
+
flag) · --description <text>
|
|
722
715
|
`
|
|
723
716
|
};
|
|
724
717
|
|
package/src/council/debate.js
CHANGED
|
@@ -8,12 +8,17 @@
|
|
|
8
8
|
* new tally input plus the debate.json findings rows. run.js then re-runs
|
|
9
9
|
* tally() on the returned input and decorates the record. Keeps run.js under the
|
|
10
10
|
* line gate.
|
|
11
|
+
* Also holds the pure pre-debate helpers (target selection, dispute detection, re-vote bundling) moved from run-debate.js (v4.7 PR0).
|
|
11
12
|
*/
|
|
12
13
|
|
|
13
14
|
const PAST_TENSE = { defend: 'defended', amend: 'amended', withdraw: 'withdrawn', 'no-response': 'no-response' };
|
|
14
15
|
|
|
15
|
-
//
|
|
16
|
-
//
|
|
16
|
+
// The debate-role vocabulary: a debate leg is an extra leg by an already-benched model,
|
|
17
|
+
// never an extra ledger row and never that model's ledger identity. Through v4.6 this Set
|
|
18
|
+
// ALSO drove ledger.js's join skip-set directly; Task 7 (v4.7 D4) replaced that with
|
|
19
|
+
// ledger.js's own LEDGER_JOIN_ROLES allowlist (fail-closed: everything not named there is
|
|
20
|
+
// excluded, not just DEBATE_ROLES), so this Set no longer has any runtime consumer outside
|
|
21
|
+
// this module — kept exported because debate.test.js pins its exact contents.
|
|
17
22
|
const DEBATE_ROLES = new Set(['rebuttal', 'revote']);
|
|
18
23
|
|
|
19
24
|
/**
|
|
@@ -78,21 +83,95 @@ function decorateRecord(record, debateFindings) {
|
|
|
78
83
|
}
|
|
79
84
|
|
|
80
85
|
/**
|
|
81
|
-
* runStats rows for the debate legs (spec §5.5)
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
+
* runStats rows for the debate legs (spec §5.5), plus v4.7 D2/E4's row-per-launch
|
|
87
|
+
* extras: role is 'rebuttal' | 'revote' for the primary defense/re-vote legs,
|
|
88
|
+
* 'superseded' for an original leg a successful repair replaced, and 'repair' for
|
|
89
|
+
* a repair attempt that itself never became usable (error status rides naturally
|
|
90
|
+
* off the raw leg). The rebuttal/revote legs never enter meta.models, so the
|
|
91
|
+
* ledger stays one row per (run×model). DEBATE_ROLES remains the debate-role
|
|
92
|
+
* vocabulary (rebuttal/revote); the ledger's overwrite protection for ALL FOUR
|
|
93
|
+
* of these row-per-launch roles — rebuttal, revote, superseded AND repair —
|
|
94
|
+
* lives in ledger.js's own LEDGER_JOIN_ROLES allowlist (v4.7 D4, Task 7):
|
|
95
|
+
* a role not named there never joins, full stop, regardless of which module
|
|
96
|
+
* produced the row or whether it is even in DEBATE_ROLES.
|
|
97
|
+
* @param {{defenseLegs: Array, revoteLegs: Array, supersededLegs?: Array,
|
|
98
|
+
* repairLegs?: Array}} args leg metadata
|
|
86
99
|
* @returns {Array<object>}
|
|
87
100
|
*/
|
|
88
|
-
function debateRunStatsRows({ defenseLegs, revoteLegs }) {
|
|
101
|
+
function debateRunStatsRows({ defenseLegs, revoteLegs, supersededLegs, repairLegs }) {
|
|
89
102
|
const mk = (role) => (l) => ({
|
|
90
103
|
model: l.model, role, wasChair: false, conformance: l.conformance || 'clean',
|
|
91
104
|
status: l.status || 'unknown',
|
|
92
105
|
durationMs: typeof l.durationMs === 'number' ? l.durationMs : null,
|
|
93
106
|
usage: l.usage || null,
|
|
107
|
+
...(l.waveId ? { waveId: l.waveId } : {}),
|
|
108
|
+
...(l.resolvedModel ? { resolvedModel: l.resolvedModel } : {}),
|
|
94
109
|
});
|
|
95
|
-
return [
|
|
110
|
+
return [
|
|
111
|
+
...(defenseLegs || []).map(mk('rebuttal')),
|
|
112
|
+
...(revoteLegs || []).map(mk('revote')),
|
|
113
|
+
...(supersededLegs || []).map(mk('superseded')),
|
|
114
|
+
...(repairLegs || []).map(mk('repair')),
|
|
115
|
+
];
|
|
96
116
|
}
|
|
97
117
|
|
|
98
|
-
|
|
118
|
+
/** Spec §5.7 fallback: a dead/unparseable defense means every bundled id's original stands. */
|
|
119
|
+
function allNoResponse(ids) {
|
|
120
|
+
const byId = {};
|
|
121
|
+
for (const id of ids) { byId[id] = { action: 'no-response' }; }
|
|
122
|
+
return byId;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** True when there is nothing to challenge (spec §5.1). */
|
|
126
|
+
function nothingToDebate(provisionalRecord) {
|
|
127
|
+
if (!provisionalRecord || provisionalRecord.judged === false) { return true; }
|
|
128
|
+
const n = provisionalRecord.findings.filter(f => f.tier === 'Contested' || f.tier === 'Disputed').length;
|
|
129
|
+
return n === 0;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Judges whose provisional adjudications dispute at least one bundled id. */
|
|
133
|
+
function disputingJudges(provisionalRecord, bundledIds) {
|
|
134
|
+
const ids = new Set(bundledIds);
|
|
135
|
+
const judges = new Set();
|
|
136
|
+
for (const f of provisionalRecord.findings) {
|
|
137
|
+
if (!ids.has(f.id)) { continue; }
|
|
138
|
+
for (const adj of f.adjudications || []) {
|
|
139
|
+
if (adj.verdict === 'dispute') { judges.add(adj.judge); }
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return [...judges];
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Group Contested+Disputed findings by raiser (defense targets). */
|
|
146
|
+
function debateTargets(provisionalRecord, tallyInput) {
|
|
147
|
+
const claimById = new Map(tallyInput.findings.map(f => [f.id, f]));
|
|
148
|
+
const byRaiser = {};
|
|
149
|
+
const previousTier = {};
|
|
150
|
+
for (const f of provisionalRecord.findings) {
|
|
151
|
+
if (f.tier !== 'Contested' && f.tier !== 'Disputed') { continue; }
|
|
152
|
+
previousTier[f.id] = f.tier;
|
|
153
|
+
const src = claimById.get(f.id) || {};
|
|
154
|
+
const peerVerdicts = (f.adjudications || []).filter(a => a.judge !== f.raiser).map(a => a.verdict);
|
|
155
|
+
(byRaiser[f.raiser] = byRaiser[f.raiser] || []).push({ id: f.id, claim: src.claim,
|
|
156
|
+
severity: f.severity, location: src.location, peerVerdicts, disputeReasons: [] });
|
|
157
|
+
}
|
|
158
|
+
return { byRaiser, previousTier };
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** The re-vote bundle: defended-or-amended findings ONLY (spec §5.1 — withdrawn never appear). */
|
|
162
|
+
function bundleFor(defenseResults, tallyInput) {
|
|
163
|
+
const out = [];
|
|
164
|
+
for (const dr of defenseResults) {
|
|
165
|
+
for (const [id, resp] of Object.entries(dr.byId)) {
|
|
166
|
+
if (resp.action !== 'defend' && resp.action !== 'amend') { continue; }
|
|
167
|
+
const src = tallyInput.findings.find(f => f.id === id) || {};
|
|
168
|
+
out.push({ id, severity: src.severity, amended: resp.action === 'amend',
|
|
169
|
+
claim: resp.action === 'amend' ? resp.claim : src.claim,
|
|
170
|
+
argument: resp.argument || 'defended without extra argument' });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return out;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
module.exports = { applyDebate, decorateRecord, debateRunStatsRows, PAST_TENSE, DEBATE_ROLES,
|
|
177
|
+
allNoResponse, nothingToDebate, disputingJudges, debateTargets, bundleFor };
|
package/src/council/ledger.js
CHANGED
|
@@ -3,11 +3,55 @@
|
|
|
3
3
|
const fs = require('fs');
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const { getConfigDir } = require('../utils/config');
|
|
6
|
-
const { DEBATE_ROLES } = require('./debate');
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
// v4.7 GOA-7 D9: v2 rows may carry `resolvedModel` (the executable id that
|
|
8
|
+
// served, copied from the joined runStats row, emit-only-when-set). Absent
|
|
9
|
+
// resolvedModel ⇒ legacy row, aggregated under its alias (spec R2) — this
|
|
10
|
+
// covers ALL pre-v2 history AND leg-less v2 rows (give-up chair, dead seats,
|
|
11
|
+
// claude, hand-assembled tally input), whose resolution is genuinely
|
|
12
|
+
// unknowable. Legacy-READ only: readers never inspect schemaVersion, rows are
|
|
13
|
+
// never migrated.
|
|
14
|
+
const LEDGER_SCHEMA_VERSION = 2;
|
|
9
15
|
const LEDGER_FILE = 'council-ledger.jsonl';
|
|
10
16
|
|
|
17
|
+
// v4.7 D4/E1/E2/E6 (Task-7, task-6/task-7 adjudications): fail-closed
|
|
18
|
+
// ALLOWLIST of runStats roles the ledger join (below) may consume as a
|
|
19
|
+
// model's ledger identity. 'council' is the legacy default role (pre-#83
|
|
20
|
+
// runs, and the av-receiver golden fixture — errata E2, must stay green).
|
|
21
|
+
// 'redteam' is the second-opinion skill's documented primary-seat role
|
|
22
|
+
// (skills/second-opinion/MANUAL-ORCHESTRATION.md:147; red-team runs record
|
|
23
|
+
// to the ledger per COUNCIL-DESIGN.md:266 — errata E6, task-7 review: without
|
|
24
|
+
// it a red-team row's role/wasChair/conformance never join, silently
|
|
25
|
+
// fabricating conformance:'clean' via the `|| 'clean'` fallback below).
|
|
26
|
+
// 'judge' stays excluded (#83's overwrite-guard: judges ARE bench models, and
|
|
27
|
+
// their Stage-2 cost-attribution row must never win over the seat row). Every
|
|
28
|
+
// OTHER non-primary row-per-launch producer — 'chair-attempt', 'repair',
|
|
29
|
+
// 'superseded', and the debate pair 'rebuttal'/'revote' — shares a model with
|
|
30
|
+
// that model's real bench row and must never join either: skipped by
|
|
31
|
+
// omission, including any future role never added here (fail-closed, not a
|
|
32
|
+
// skip-list that a new producer could silently slip past). This is the E6
|
|
33
|
+
// trade, made explicit: any free-form/custom role label a future producer
|
|
34
|
+
// invents is rejected BY DESIGN until someone deliberately adds it here —
|
|
35
|
+
// the allowlist would rather silently drop a legitimate new role's
|
|
36
|
+
// role/wasChair/conformance (falling back to 'council'/false/'clean') than
|
|
37
|
+
// ever let an unreviewed role win the model-keyed join.
|
|
38
|
+
//
|
|
39
|
+
// Final-review consolidated wave (owner-ruled): the ABSENCE of a role
|
|
40
|
+
// (null/undefined) is a DIFFERENT case from a NAMED-unknown role and joins
|
|
41
|
+
// too — this is the docs/council.md:562-blessed hand-assembled tally-input
|
|
42
|
+
// shape ("the legacy default `council` … pre-#83 rows, or hand-assembled
|
|
43
|
+
// tally input that never set a role"), and mirrors GOA-7's absent-field⇒
|
|
44
|
+
// legacy pattern elsewhere in this codebase: a field that was never set gets
|
|
45
|
+
// treated as the oldest/legacy shape, not silently dropped like an
|
|
46
|
+
// unreviewed value would be. A NAMED custom label (e.g. 'custom-thing') is
|
|
47
|
+
// still rejected exactly as E6 describes — only the missing-field case is
|
|
48
|
+
// legacy; an actively wrong or unreviewed one is not.
|
|
49
|
+
const LEDGER_JOIN_ROLES = new Set(['seat', 'critic', 'chair', 'claude', 'council', 'redteam']);
|
|
50
|
+
function joinsLedger(role) {
|
|
51
|
+
return role === null || role === undefined ||
|
|
52
|
+
LEDGER_JOIN_ROLES.has(role) || (typeof role === 'string' && role.startsWith('lens:'));
|
|
53
|
+
}
|
|
54
|
+
|
|
11
55
|
function countSeverity(findings) {
|
|
12
56
|
const c = { blocker: 0, major: 0, minor: 0, nit: 0 };
|
|
13
57
|
for (const f of findings) { if (c[f.severity] !== undefined) { c[f.severity] += 1; } }
|
|
@@ -18,10 +62,10 @@ function countSeverity(findings) {
|
|
|
18
62
|
function buildLedgerRows(record) {
|
|
19
63
|
const { meta, findings, streetCred, runStats, judged } = record;
|
|
20
64
|
const sc = new Map(streetCred.map(s => [s.model, s]));
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
const rs = new Map(runStats.filter(r =>
|
|
65
|
+
// The join below is keyed by MODEL — only allowlisted roles (joinsLedger,
|
|
66
|
+
// above) may win it, so a non-primary row-per-launch row can never
|
|
67
|
+
// silently overwrite a model's real bench (seat) row.
|
|
68
|
+
const rs = new Map(runStats.filter(r => joinsLedger(r.role))
|
|
25
69
|
.map(r => [r.model, r]));
|
|
26
70
|
return meta.models.map(model => {
|
|
27
71
|
const raised = findings.filter(f => f.raiser === model);
|
|
@@ -39,6 +83,7 @@ function buildLedgerRows(record) {
|
|
|
39
83
|
confirmRate: judged && denom ? raised.filter(f => f.tier === 'Confirmed').length / denom : null,
|
|
40
84
|
factErrorRate: judged && denom ? raised.filter(f => f.tier === 'Disputed').length / denom : null,
|
|
41
85
|
conformance: r.conformance || 'clean',
|
|
86
|
+
...(r.resolvedModel ? { resolvedModel: r.resolvedModel } : {}),
|
|
42
87
|
};
|
|
43
88
|
});
|
|
44
89
|
}
|
|
@@ -61,25 +106,41 @@ function readRows(dir) {
|
|
|
61
106
|
|
|
62
107
|
function avg(nums) { return nums.length ? nums.reduce((s, x) => s + x, 0) / nums.length : null; }
|
|
63
108
|
|
|
64
|
-
/**
|
|
109
|
+
/**
|
|
110
|
+
* Aggregate the ledger per model. peersOnly nulls excluded; lowN flags < 3 runs.
|
|
111
|
+
* v4.7 GOA-7 D10: groups by `row.resolvedModel || row.model` — v2 rows segment
|
|
112
|
+
* by the executable id that actually served; rows without a resolvedModel
|
|
113
|
+
* (pre-v2 history, leg-less rows, hand-assembled tally input) stay alias-keyed
|
|
114
|
+
* with `legacy: true`. `aliases` lists every row-level `model` (alias) observed
|
|
115
|
+
* for the group, most recently observed FIRST — ledger append order is the only
|
|
116
|
+
* recency signal (`date` is day-granular, free-form on the MCP path), so
|
|
117
|
+
* aliases[0] is the launch-preferred name (pickFallbackChair, D11).
|
|
118
|
+
* Version-blind by design: schemaVersion is never read (legacy-read, R2).
|
|
119
|
+
*/
|
|
65
120
|
function deriveReliability(opts = {}) {
|
|
66
121
|
const dir = opts.dir || getConfigDir();
|
|
67
|
-
const
|
|
122
|
+
const byKey = new Map();
|
|
68
123
|
for (const row of readRows(dir)) {
|
|
69
|
-
|
|
70
|
-
|
|
124
|
+
const key = row.resolvedModel || row.model;
|
|
125
|
+
if (!byKey.has(key)) { byKey.set(key, []); }
|
|
126
|
+
byKey.get(key).push(row);
|
|
71
127
|
}
|
|
72
|
-
return [...
|
|
128
|
+
return [...byKey.entries()].map(([model, rows]) => {
|
|
73
129
|
const peers = rows.map(r => r.streetCredPeersOnly).filter(v => typeof v === 'number');
|
|
74
130
|
const confirms = rows.map(r => r.confirmRate).filter(v => typeof v === 'number');
|
|
75
131
|
const facts = rows.map(r => r.factErrorRate).filter(v => typeof v === 'number');
|
|
76
132
|
const conformance = rows.reduce((acc, r) => { acc[r.conformance] = (acc[r.conformance] || 0) + 1; return acc; }, {});
|
|
133
|
+
const lastSeen = new Map();
|
|
134
|
+
rows.forEach((r, i) => { lastSeen.set(r.model, i); });
|
|
135
|
+
const aliases = [...lastSeen.entries()].sort((a, b) => b[1] - a[1]).map(([m]) => m);
|
|
77
136
|
return {
|
|
78
137
|
model, runs: rows.length, lowN: rows.length < 3,
|
|
79
138
|
avgStreetCredPeersOnly: avg(peers),
|
|
80
139
|
lifetimeConfirmRate: avg(confirms),
|
|
81
140
|
lifetimeFactErrorRate: avg(facts),
|
|
82
141
|
conformance,
|
|
142
|
+
aliases,
|
|
143
|
+
...(rows.every(r => !r.resolvedModel) ? { legacy: true } : {}),
|
|
83
144
|
};
|
|
84
145
|
});
|
|
85
146
|
}
|
|
@@ -45,18 +45,22 @@ function runSave(name, modelsArg, useJson) {
|
|
|
45
45
|
hint: 'each member must be a known alias (see `amicus models`) or a provider/model id containing "/"' });
|
|
46
46
|
}
|
|
47
47
|
const overwritten = !!getCouncil(name);
|
|
48
|
+
const shadowsBuiltin = listBuiltinCouncilNames().includes(name);
|
|
48
49
|
const cfg = loadConfig() || {};
|
|
49
50
|
if (!cfg.councils) { cfg.councils = {}; }
|
|
50
51
|
cfg.councils[name] = members;
|
|
51
52
|
saveConfig(cfg);
|
|
52
|
-
const doc = { ok: true, name, models: members, overwritten };
|
|
53
|
+
const doc = { ok: true, name, models: members, overwritten, shadowsBuiltin };
|
|
53
54
|
process.stdout.write(useJson ? JSON.stringify(doc, null, 2) + '\n' : renderSave(doc));
|
|
54
55
|
return 0;
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
function renderSave(doc) {
|
|
58
59
|
const notice = doc.overwritten ? ' (overwritten)' : '';
|
|
59
|
-
|
|
60
|
+
const shadow = doc.shadowsBuiltin
|
|
61
|
+
? ` note: '${doc.name}' now shadows the built-in bench of the same name — 'amicus council list' shows both\n`
|
|
62
|
+
: '';
|
|
63
|
+
return `Saved council '${doc.name}'${notice}: ${doc.models.join(', ')}\n` + shadow +
|
|
60
64
|
" for full run configuration — chair, options, templates — see 'amicus pack'\n";
|
|
61
65
|
}
|
|
62
66
|
|
package/src/council/report.js
CHANGED
|
@@ -66,13 +66,24 @@ function toModel(verdict, wave) {
|
|
|
66
66
|
.map(f => ({ id: f.id, previousTier: f.debate.previousTier, tier: f.tier })),
|
|
67
67
|
};
|
|
68
68
|
const runStats = verdict.runStats || [];
|
|
69
|
-
// Cost-row role tag (Plan 2 final review F1): #83 gave
|
|
70
|
-
// runStats row, so a bench model can now appear twice
|
|
71
|
-
// indistinguishable by `model` alone.
|
|
72
|
-
//
|
|
73
|
-
// their
|
|
69
|
+
// Cost-row role tag (Plan 2 final review F1, extended v4.7 D6): #83 gave
|
|
70
|
+
// judges their own runStats row, so a bench model can now appear twice
|
|
71
|
+
// (seat + judge), indistinguishable by `model` alone. v4.7's row-per-launch
|
|
72
|
+
// producers (chair-attempt/repair/superseded) create the exact same
|
|
73
|
+
// collision for their model. Tag ONLY these four roles — old verdicts have
|
|
74
|
+
// none of them, so chair/critic/lens/seat rows stay byte-identical to their
|
|
75
|
+
// historical rendering (report.test.js:189-199 pins the judge case exactly).
|
|
76
|
+
// Object.create(null): a plain `{...}` literal inherits Object.prototype, so a role
|
|
77
|
+
// literally named 'constructor'/'toString'/etc would resolve to an inherited (truthy)
|
|
78
|
+
// function via bracket lookup instead of `undefined` — silently corrupting that row's
|
|
79
|
+
// rendered model label. A null-prototype object has no inherited keys to collide with.
|
|
80
|
+
const ROLE_SUFFIX = Object.create(null);
|
|
81
|
+
ROLE_SUFFIX.judge = 'judge';
|
|
82
|
+
ROLE_SUFFIX['chair-attempt'] = 'chair-attempt';
|
|
83
|
+
ROLE_SUFFIX.repair = 'repair';
|
|
84
|
+
ROLE_SUFFIX.superseded = 'superseded';
|
|
74
85
|
const costRows = runStats.map(r => ({
|
|
75
|
-
model: r.role
|
|
86
|
+
model: ROLE_SUFFIX[r.role] ? `${r.model} (${ROLE_SUFFIX[r.role]})` : r.model,
|
|
76
87
|
status: r.status, durationMs: r.durationMs,
|
|
77
88
|
cost: r.usage && r.usage.cost ? r.usage.cost : null,
|
|
78
89
|
}));
|
|
@@ -40,6 +40,9 @@ function worseConformance(a, b) {
|
|
|
40
40
|
* leg doc yields durationMs/usage null (never invent a value). `model` (the
|
|
41
41
|
* council alias) overrides leg.model (the resolved executable id) so ledger
|
|
42
42
|
* rows join meta.models by exact string (ledger.js:20-24).
|
|
43
|
+
* `resolvedModel` (v4.7 GOA-7) preserves leg.model — the executable id that
|
|
44
|
+
* actually served, post-fallback-substitution — emit-only-when-set and never
|
|
45
|
+
* sourced from modelInput (an alias must never masquerade as a resolved id).
|
|
43
46
|
*
|
|
44
47
|
* ⚠️ LC-11 / review F1: `findingsUnverified` and `repairRefused` are the same
|
|
45
48
|
* class of fact as `conformance` and ride the same row. They are the two halves
|
|
@@ -60,6 +63,8 @@ function buildRunStatsEntry({ leg, model, role, wasChair, conformance, findingsU
|
|
|
60
63
|
conformance: conformance || 'clean',
|
|
61
64
|
...(findingsUnverified ? { findingsUnverified: true } : {}),
|
|
62
65
|
...(repairRefused ? { repairRefused } : {}),
|
|
66
|
+
...(leg && leg.waveId ? { waveId: leg.waveId } : {}),
|
|
67
|
+
...(leg && leg.model ? { resolvedModel: leg.model } : {}),
|
|
63
68
|
status: leg ? leg.status : 'error',
|
|
64
69
|
durationMs: leg && typeof leg.durationMs === 'number' ? leg.durationMs : null,
|
|
65
70
|
usage: (leg && leg.usage) || null,
|
|
@@ -131,14 +136,17 @@ function claudeRunStatsRow() {
|
|
|
131
136
|
* @param {{runId: string, date: string, bench: string[], chair: string,
|
|
132
137
|
* reviews: Array<{model, role, conformance, leg, globalFindings}>,
|
|
133
138
|
* judgeResults: Array<{judge, ok, order, adjudications}>,
|
|
134
|
-
* chairStats: object|null, claudeReview?: object|null}} args
|
|
139
|
+
* chairStats: object|null, claudeReview?: object|null, extraRows?: Array<object>}} args
|
|
135
140
|
* `claudeReview` (v4.1 §4.4) amends the v4.0 meta pin: present ⇒ claudeInCouncil
|
|
136
141
|
* true, 'claude' joins meta.models (the street-cred universe), its findings join
|
|
137
142
|
* the pool and it gets the synthesized null-usage runStats row. Absent ⇒ v4.0
|
|
138
|
-
* output byte-for-byte.
|
|
143
|
+
* output byte-for-byte. `extraRows` (v4.7 D2/E4) are pre-built runStats rows
|
|
144
|
+
* (repair/superseded/dead-seat-error, from runStage1 today) appended right
|
|
145
|
+
* after the primary review rows, before judge/chair accounting — absent or
|
|
146
|
+
* empty ⇒ byte-for-byte unchanged, so the pre-v4.7 length-7 pins stay green.
|
|
139
147
|
*/
|
|
140
148
|
function buildTallyInput({ runId, date, bench, chair, reviews, judgeResults, chairStats,
|
|
141
|
-
claudeReview }) {
|
|
149
|
+
claudeReview, extraRows }) {
|
|
142
150
|
const meta = {
|
|
143
151
|
runId, date, runType: 'headless',
|
|
144
152
|
models: bench.slice(), // bench seats exactly: critic included, chair excluded
|
|
@@ -154,6 +162,10 @@ function buildTallyInput({ runId, date, bench, chair, reviews, judgeResults, cha
|
|
|
154
162
|
leg: r.leg, model: r.model, role: r.role, wasChair: false, conformance: r.conformance,
|
|
155
163
|
findingsUnverified: r.findingsUnverified, repairRefused: r.repairRefused,
|
|
156
164
|
}));
|
|
165
|
+
// v4.7 D2/E4: pre-built rows (repair/superseded/dead-seat-error) ride right
|
|
166
|
+
// after the primary review rows — same "primary-adjacent" shape, just not
|
|
167
|
+
// sourced from a surviving review. Absent/empty ⇒ no-op (pre-v4.7 byte parity).
|
|
168
|
+
runStats.push(...(extraRows || []));
|
|
157
169
|
if (claudeReview) {
|
|
158
170
|
meta.models.push(CLAUDE_SEAT); // last, mirroring its review-N+1 label
|
|
159
171
|
meta.claudeInCouncil = true;
|
|
@@ -153,9 +153,9 @@ function createBudget({ allLegs, maxCost, runDir, degrade, write }) {
|
|
|
153
153
|
channel: 'budget-refusal',
|
|
154
154
|
what: `wave ${rec.waveId} (${rec.models.join(', ') || 'no models'}) — those seats DID NOT `
|
|
155
155
|
+ 'LAUNCH and are missing from this council',
|
|
156
|
-
why: `the $${maxCost}
|
|
156
|
+
why: `the $${maxCost} cost ceiling for this run refused it${message ? `: ${message}` : ''}`,
|
|
157
157
|
effect: 'The run continues with the bench that did launch and will exit degraded (2)',
|
|
158
|
-
remedy:
|
|
158
|
+
remedy: "Raise this run's cost ceiling, or turn the cost gate off, to seat them",
|
|
159
159
|
});
|
|
160
160
|
if (runDir) {
|
|
161
161
|
// Never let bookkeeping sink a run that is otherwise fine.
|