amicus 3.2.3 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +106 -0
- package/README.md +15 -3
- package/electron/main.js +4 -1
- package/package.json +3 -1
- package/schemas/abort.schema.json +17 -0
- package/schemas/alias-audit.schema.json +17 -0
- package/schemas/council-run.schema.json +38 -0
- package/schemas/council-stats.schema.json +28 -0
- package/schemas/council-tally.schema.json +70 -0
- package/schemas/council-validate.schema.json +22 -0
- package/schemas/council-verdict.schema.json +47 -0
- package/schemas/doctor.schema.json +29 -0
- package/schemas/error.schema.json +23 -0
- package/schemas/model-catalog.schema.json +19 -0
- package/schemas/run.schema.json +26 -0
- package/schemas/spend.schema.json +16 -0
- package/schemas/wave.schema.json +33 -0
- package/skills/second-opinion/SEAT-BRIEFS.md +5 -3
- package/skills/second-opinion/SKILL.md +8 -0
- package/src/cli-handlers-abort.js +29 -0
- package/src/cli-handlers-council-run.js +168 -0
- package/src/cli-handlers-council.js +8 -5
- package/src/cli-handlers-status.js +35 -4
- package/src/cli.js +9 -0
- package/src/council/anonymize.js +76 -0
- package/src/council/briefings-stage2.js +150 -0
- package/src/council/briefings.js +141 -0
- package/src/council/findings.js +13 -1
- package/src/council/ledger.js +13 -1
- package/src/council/parse-stage2.js +103 -0
- package/src/council/run-assemble.js +100 -0
- package/src/council/run-launch.js +99 -0
- package/src/council/run-stages.js +203 -0
- package/src/council/run-state.js +161 -0
- package/src/council/run.js +277 -0
- package/src/council/tally.js +3 -1
- package/src/council/verdict.js +9 -2
- package/src/headless.js +24 -25
- package/src/mcp-council-awareness.js +187 -0
- package/src/mcp-council-run.js +161 -0
- package/src/mcp-server.js +87 -28
- package/src/mcp-tools.js +50 -0
- package/src/prompt-builder.js +36 -19
- package/src/sidecar/fanout-leg.js +2 -2
- package/src/sidecar/fanout.js +1 -1
- package/src/sidecar/resume.js +7 -2
- package/src/utils/abort-result.js +1 -1
- package/src/utils/error-doc.js +2 -0
- package/src/utils/fold-marker.js +21 -0
- package/src/utils/route-error.js +26 -0
- package/src/utils/start-helpers.js +19 -10
- package/src/utils/untrusted-fence.js +8 -7
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// src/council/briefings-stage2.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module council/briefings-stage2
|
|
6
|
+
* Pure Stage-2 (judge bundle) and Stage-3 (chair packet) template builders
|
|
7
|
+
* for the headless council engine (spec §5). Split from ./briefings.js for
|
|
8
|
+
* the 300-line gate. The bundle is identical for every judge and never leaks
|
|
9
|
+
* seat/lens/critic information or model names (skill §5.1 / Stage-2 rule).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const JUDGE_NO_TOOLS_PREAMBLE =
|
|
13
|
+
'Do NOT use any tools or read any files; everything is in this message; ' +
|
|
14
|
+
'begin immediately with A1:';
|
|
15
|
+
|
|
16
|
+
const CHAIR_NO_TOOLS_PREAMBLE =
|
|
17
|
+
'Do NOT use any tools or read any files; everything is in this message; ' +
|
|
18
|
+
'begin immediately with the verdict.';
|
|
19
|
+
|
|
20
|
+
const CHAIR_VERDICT_VALUES = ['Ship it', 'Fix these first', 'Fundamental rethink'];
|
|
21
|
+
|
|
22
|
+
/** Stage-2 headless output contract (spec §5, embedded in the judge bundle). */
|
|
23
|
+
const JUDGE_OUTPUT_CONTRACT = [
|
|
24
|
+
'End your response with a trailing fenced ```json block — no text after it — in',
|
|
25
|
+
'exactly this shape:',
|
|
26
|
+
'',
|
|
27
|
+
'```json',
|
|
28
|
+
'{',
|
|
29
|
+
' "ranking": ["Review B", "Review A", "Review C"],',
|
|
30
|
+
' "adjudications": [',
|
|
31
|
+
' { "id": "A1", "verdict": "agree" },',
|
|
32
|
+
' { "id": "B2", "verdict": "dispute" }',
|
|
33
|
+
' ]',
|
|
34
|
+
'}',
|
|
35
|
+
'```',
|
|
36
|
+
'',
|
|
37
|
+
'- "ranking": every review label below, ordered most to least accurate and insightful.',
|
|
38
|
+
' Ties: use a nested array for tied labels, e.g. [["Review A", "Review B"], "Review C"].',
|
|
39
|
+
'- "adjudications": one entry per listed finding id; "verdict" is one of:',
|
|
40
|
+
' agree | dispute | neutral. An "I missed this — it\'s valid" counts as agree.',
|
|
41
|
+
].join('\n');
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The single shared anonymized judge bundle.
|
|
45
|
+
* @param {{reviews: Array<{label: string, text: string}>,
|
|
46
|
+
* findings: Array<{id: string, severity: string, claim: string}>}} args
|
|
47
|
+
*/
|
|
48
|
+
function buildJudgeBundle({ reviews, findings }) {
|
|
49
|
+
const findingLines = findings.map(f => `${f.id} [${f.severity}] ${f.claim}`).join('\n');
|
|
50
|
+
const reviewBlocks = reviews
|
|
51
|
+
.map(r => `--- ${r.label} ---\n${r.text}`)
|
|
52
|
+
.join('\n\n');
|
|
53
|
+
return [
|
|
54
|
+
JUDGE_NO_TOOLS_PREAMBLE,
|
|
55
|
+
'You are judging the anonymized peer reviews below. Do two things:',
|
|
56
|
+
'Task A — Rank: order the reviews from most to least accurate and insightful.',
|
|
57
|
+
'Task B — Adjudicate: for EVERY finding id listed below, state agree, dispute, or ' +
|
|
58
|
+
'neutral with your reasoning in prose.',
|
|
59
|
+
JUDGE_OUTPUT_CONTRACT,
|
|
60
|
+
'--- FINDINGS INDEX (run-global ids) ---',
|
|
61
|
+
findingLines,
|
|
62
|
+
reviewBlocks,
|
|
63
|
+
].join('\n\n');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Bounded judge-repair re-prompt (solo; ≤ 2 per judge — spec §5). */
|
|
67
|
+
function buildJudgeRepairPrompt({ errors }) {
|
|
68
|
+
const lines = (errors || []).map(e => `- ${e.code}: ${e.detail}`).join('\n');
|
|
69
|
+
return [
|
|
70
|
+
'Do NOT use any tools or read any files; everything is in this message; begin ' +
|
|
71
|
+
'immediately with the JSON block.',
|
|
72
|
+
'Your previous judging response\'s trailing JSON failed validation with these errors:',
|
|
73
|
+
lines,
|
|
74
|
+
'Re-emit ONLY the corrected JSON block as a single fenced ```json block:',
|
|
75
|
+
JUDGE_OUTPUT_CONTRACT,
|
|
76
|
+
].join('\n\n');
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Verdict-scale addendum (SEAT-BRIEFS.md § Chair verdict-scale addendum; always on headless). */
|
|
80
|
+
const VERDICT_SCALE_ADDENDUM = [
|
|
81
|
+
'After your synthesis, add two closing sections:',
|
|
82
|
+
'',
|
|
83
|
+
'1. HARD QUESTIONS — three to five questions the material\'s author has probably not',
|
|
84
|
+
' asked themselves, chosen so that an unanswerable question reveals a structural gap',
|
|
85
|
+
' (not gotchas — questions whose answers should exist).',
|
|
86
|
+
'2. A final line, alone on the last line, containing ONLY the phrase — no rationale, no',
|
|
87
|
+
' dash, no trailing text of any kind — exactly one of:',
|
|
88
|
+
'',
|
|
89
|
+
' VERDICT: Ship it',
|
|
90
|
+
' VERDICT: Fix these first',
|
|
91
|
+
' VERDICT: Fundamental rethink',
|
|
92
|
+
'',
|
|
93
|
+
' Pick one. "Ship it" = solid, nothing blocking. "Fix these first" = specific gaps',
|
|
94
|
+
' must be resolved first. "Fundamental rethink" = structural problems that cannot be',
|
|
95
|
+
' patched. Name the gaps or the structural problems in the synthesis ABOVE, not on the',
|
|
96
|
+
' VERDICT line itself — that line carries the phrase and nothing else.',
|
|
97
|
+
].join('\n');
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* De-anonymized chair packet (spec §5/§6: the chair sees identities).
|
|
101
|
+
* @param {{reviews: Array<{model: string, text: string}>,
|
|
102
|
+
* rankings: Array<{judge: string, order: Array<string|string[]>}>,
|
|
103
|
+
* adjudications: Array<{findingId: string, judge: string, verdict: string}>,
|
|
104
|
+
* tierCounts: object}} args
|
|
105
|
+
*/
|
|
106
|
+
function buildChairPacket({ reviews, rankings, adjudications, tierCounts }) {
|
|
107
|
+
const reviewBlocks = reviews.map(r => `--- Review by ${r.model} ---\n${r.text}`).join('\n\n');
|
|
108
|
+
const rankingLines = rankings
|
|
109
|
+
.map(r => `${r.judge}: ${JSON.stringify(r.order)}`)
|
|
110
|
+
.join('\n');
|
|
111
|
+
const adjLines = adjudications
|
|
112
|
+
.map(a => `${a.findingId} — ${a.judge}: ${a.verdict}`)
|
|
113
|
+
.join('\n');
|
|
114
|
+
const tiers = JSON.stringify(tierCounts);
|
|
115
|
+
return [
|
|
116
|
+
CHAIR_NO_TOOLS_PREAMBLE,
|
|
117
|
+
'You are the council chair. Write the synthesized verdict across the reviews, ' +
|
|
118
|
+
'rankings, and adjudications below. Weigh each reviewer\'s findings by their ' +
|
|
119
|
+
'peer-validated standing (rank position and adjudication pattern), distinguish ' +
|
|
120
|
+
'findings the bench broadly endorsed from contested or singleton claims, and ' +
|
|
121
|
+
'arrive at an overall assessment of the material.',
|
|
122
|
+
`Deterministic tier counts (peers-only cascade): ${tiers}`,
|
|
123
|
+
'--- STAGE-1 REVIEWS (de-anonymized) ---',
|
|
124
|
+
reviewBlocks,
|
|
125
|
+
'--- PEER RANKINGS (judge: order, best first) ---',
|
|
126
|
+
rankingLines,
|
|
127
|
+
'--- PER-FINDING ADJUDICATIONS ---',
|
|
128
|
+
adjLines,
|
|
129
|
+
VERDICT_SCALE_ADDENDUM,
|
|
130
|
+
].join('\n\n');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** One-shot chair repair: the VERDICT line was missing (spec §5 chair contract). */
|
|
134
|
+
function buildChairRepairPrompt() {
|
|
135
|
+
return [
|
|
136
|
+
'Do NOT use any tools or read any files; everything is in this message; begin ' +
|
|
137
|
+
'immediately with the VERDICT line.',
|
|
138
|
+
'Your synthesis was received, but the final parseable line was missing. Emit ONLY ' +
|
|
139
|
+
'one line, exactly one of:',
|
|
140
|
+
'VERDICT: Ship it',
|
|
141
|
+
'VERDICT: Fix these first',
|
|
142
|
+
'VERDICT: Fundamental rethink',
|
|
143
|
+
].join('\n\n');
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
module.exports = {
|
|
147
|
+
JUDGE_NO_TOOLS_PREAMBLE, CHAIR_NO_TOOLS_PREAMBLE, CHAIR_VERDICT_VALUES,
|
|
148
|
+
JUDGE_OUTPUT_CONTRACT, VERDICT_SCALE_ADDENDUM,
|
|
149
|
+
buildJudgeBundle, buildJudgeRepairPrompt, buildChairPacket, buildChairRepairPrompt,
|
|
150
|
+
};
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// src/council/briefings.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module council/briefings
|
|
6
|
+
* Pure Stage-1 briefing template builders for the headless council engine
|
|
7
|
+
* (spec §5), adapted from skills/second-opinion/SEAT-BRIEFS.md with stricter
|
|
8
|
+
* headless output contracts. No IO, no model calls. Stage-2 / chair templates
|
|
9
|
+
* live in ./briefings-stage2.js (300-line gate split).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/** SEAT-BRIEFS.md standard anti-sycophancy clause — verbatim, EVERY Stage-1 briefing. */
|
|
13
|
+
const ANTI_SYCOPHANCY_CLAUSE =
|
|
14
|
+
'Do not soften findings to be agreeable. Lead with your most severe finding. ' +
|
|
15
|
+
'No praise cushions before criticism, and never perform enthusiasm you don\'t hold — ' +
|
|
16
|
+
'if the artifact is mediocre, say so and show why. Do not pad: report every real ' +
|
|
17
|
+
'finding and no invented ones. An empty severity category is a valid result.';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* "Produce exactly two things" framing (prose review, THEN the trailing json
|
|
21
|
+
* block). Stage-1 builders need this. The findings-repair prompt deliberately
|
|
22
|
+
* does NOT use it — a repair turn wants ONLY the corrected json, never a
|
|
23
|
+
* fresh prose review (keeping this fragment out of the repair prompt fixes
|
|
24
|
+
* the prior self-contradiction: "re-emit ONLY the JSON" followed by this
|
|
25
|
+
* "write prose then json" framing).
|
|
26
|
+
*/
|
|
27
|
+
const FINDINGS_TWO_PART_FRAMING = [
|
|
28
|
+
'Produce exactly two things, in this order:',
|
|
29
|
+
'',
|
|
30
|
+
'1. A prose review — your full narrative assessment of the material.',
|
|
31
|
+
'2. A trailing fenced ```json block immediately after the prose — no text after it:',
|
|
32
|
+
].join('\n');
|
|
33
|
+
|
|
34
|
+
/** JSON shape + field rules — shared by the full Stage-1 contract AND the repair prompt. */
|
|
35
|
+
const FINDINGS_JSON_SHAPE = [
|
|
36
|
+
'```json',
|
|
37
|
+
'{',
|
|
38
|
+
' "overall": "one-paragraph take",',
|
|
39
|
+
' "findings": [',
|
|
40
|
+
' { "id": 1, "severity": "blocker",',
|
|
41
|
+
' "claim": "…", "location": "…", "rationale": "…" }',
|
|
42
|
+
' ]',
|
|
43
|
+
'}',
|
|
44
|
+
'```',
|
|
45
|
+
'',
|
|
46
|
+
'- "id" — sequential integer within this review, starting at 1.',
|
|
47
|
+
'- "severity" — one of: blocker | major | minor | nit.',
|
|
48
|
+
'- "claim", "location", "rationale" — non-empty strings.',
|
|
49
|
+
'Emit the JSON verbatim after the prose, without preamble, so it parses cleanly.',
|
|
50
|
+
].join('\n');
|
|
51
|
+
|
|
52
|
+
/** Strict headless structured-output contract (prose + trailing ```json findings block). */
|
|
53
|
+
const FINDINGS_CONTRACT = [FINDINGS_TWO_PART_FRAMING, FINDINGS_JSON_SHAPE].join('\n\n');
|
|
54
|
+
|
|
55
|
+
/** Adapted from SEAT-BRIEFS.md § Critic seat brief (quota deliberately absent). */
|
|
56
|
+
const CRITIC_BRIEF = [
|
|
57
|
+
'You are this review bench\'s designated critic. Assume problems exist; your job is to',
|
|
58
|
+
'find them, not to confirm the material is fine. Work through four passes and fold',
|
|
59
|
+
'everything into one findings list:',
|
|
60
|
+
'',
|
|
61
|
+
'1. Adversarial pass — for every claim: what evidence supports it, or is it an',
|
|
62
|
+
' assumption presented as fact? For every goal: is it measurable? For every decision:',
|
|
63
|
+
' what alternatives were considered? For every scope boundary: real constraint, or',
|
|
64
|
+
' avoidance of hard work?',
|
|
65
|
+
'2. Edge-case hunt — walk every journey, requirement, and scenario. What happens on the',
|
|
66
|
+
' unexpected input, the failed integration, the malformed data? At zero, at one, at',
|
|
67
|
+
' scale? Report only unhandled cases.',
|
|
68
|
+
'3. Consistency check — cross-reference sections against each other: do goals have',
|
|
69
|
+
' metrics, and metrics targets? Do requirements trace back to stated needs? Does',
|
|
70
|
+
' anything contradict a stated non-goal or constraint?',
|
|
71
|
+
'4. Executability test — could someone act on this material without coming back with',
|
|
72
|
+
' clarifying questions? Wherever the answer is no, name the specific section and',
|
|
73
|
+
' exactly what is missing.',
|
|
74
|
+
'',
|
|
75
|
+
'Be specific: name the section, the line, the exact gap. Report every real finding and',
|
|
76
|
+
'no invented ones; do not pad to look thorough. An empty pass is a valid result.',
|
|
77
|
+
].join('\n');
|
|
78
|
+
|
|
79
|
+
function dateLine(date) {
|
|
80
|
+
return `Today's date is ${date}.`;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function compose(role, { briefing, date }) {
|
|
84
|
+
return [
|
|
85
|
+
role,
|
|
86
|
+
ANTI_SYCOPHANCY_CLAUSE,
|
|
87
|
+
dateLine(date),
|
|
88
|
+
FINDINGS_CONTRACT,
|
|
89
|
+
'--- MATERIAL / BRIEFING ---',
|
|
90
|
+
briefing,
|
|
91
|
+
].join('\n\n');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Standard seat briefing (Stage-1 fanout wave). */
|
|
95
|
+
function buildSeatBriefing(args) {
|
|
96
|
+
return compose(
|
|
97
|
+
'You are one reviewer on an independent multi-model review bench. Review the material ' +
|
|
98
|
+
'below against the briefing\'s own criteria.',
|
|
99
|
+
args
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Critic seat briefing (concurrent solo — spec §4 --critic). */
|
|
104
|
+
function buildCriticBriefing(args) {
|
|
105
|
+
return compose(CRITIC_BRIEF, args);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Expert-lens briefing (concurrent solo per seat — spec §4 --lenses). */
|
|
109
|
+
function buildLensBriefing({ lens, briefing, date }) {
|
|
110
|
+
return compose(
|
|
111
|
+
`Review this material strictly through the lens of a ${lens}. Raise only findings ` +
|
|
112
|
+
'that perspective is qualified to raise, at the depth a top practitioner of it would ' +
|
|
113
|
+
'reach. Stay in-domain: if something matters but is outside your lens, leave it to ' +
|
|
114
|
+
'the other reviewers.',
|
|
115
|
+
{ briefing, date }
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Bounded findings-repair re-prompt (solo; ≤ 2 per seat — SKILL.md Stage-1
|
|
121
|
+
* repair loop). References ONLY the json-shape fragment — never the
|
|
122
|
+
* "prose review THEN json" framing — so a headless model isn't handed license
|
|
123
|
+
* to write a whole new prose review on a tight repair turn.
|
|
124
|
+
*/
|
|
125
|
+
function buildFindingsRepairPrompt({ errors }) {
|
|
126
|
+
const lines = (errors || []).map(e => `- ${e.code}: ${e.detail}`).join('\n');
|
|
127
|
+
return [
|
|
128
|
+
'Do NOT use any tools or read any files; everything is in this message; begin ' +
|
|
129
|
+
'immediately with the JSON block.',
|
|
130
|
+
'Your previous review\'s trailing findings JSON failed validation with these errors:',
|
|
131
|
+
lines,
|
|
132
|
+
'Re-emit ONLY the corrected findings JSON block (the same findings, fixed — do not ' +
|
|
133
|
+
'add or remove findings), as a single fenced ```json block:',
|
|
134
|
+
FINDINGS_JSON_SHAPE,
|
|
135
|
+
].join('\n\n');
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
module.exports = {
|
|
139
|
+
ANTI_SYCOPHANCY_CLAUSE, FINDINGS_CONTRACT, FINDINGS_JSON_SHAPE, FINDINGS_TWO_PART_FRAMING,
|
|
140
|
+
buildSeatBriefing, buildCriticBriefing, buildLensBriefing, buildFindingsRepairPrompt,
|
|
141
|
+
};
|
package/src/council/findings.js
CHANGED
|
@@ -45,4 +45,16 @@ function validateFindings(jsonText) {
|
|
|
45
45
|
return { ok: errors.length === 0, findings: errors.length === 0 ? findings : [], errors };
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
/**
|
|
49
|
+
* v4.0 §7: stamp the council v2 envelope onto a validateFindings result
|
|
50
|
+
* (additive — ok/findings/errors stay top-level; existing key-readers keep
|
|
51
|
+
* working). Used by `amicus council validate --json`.
|
|
52
|
+
* @param {{ok:boolean, findings:Array, errors:Array}} result
|
|
53
|
+
* @returns {object} enveloped validate doc
|
|
54
|
+
*/
|
|
55
|
+
function buildValidateDoc(result) {
|
|
56
|
+
const { COUNCIL_SCHEMA_VERSION } = require('./tally');
|
|
57
|
+
return { schemaVersion: COUNCIL_SCHEMA_VERSION, type: 'council-validate', ...result };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
module.exports = { validateFindings, buildValidateDoc, SEVERITIES, lastJsonBlock };
|
package/src/council/ledger.js
CHANGED
|
@@ -79,4 +79,16 @@ function deriveReliability(opts = {}) {
|
|
|
79
79
|
});
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
/**
|
|
83
|
+
* v4.0 §7: wrap the deriveReliability() rows in the council v2 envelope —
|
|
84
|
+
* THE one sanctioned breaking shape change (`council stats --json` used to
|
|
85
|
+
* emit the bare array). Human rendering keeps consuming the bare rows.
|
|
86
|
+
* @param {Array<object>} models deriveReliability() output
|
|
87
|
+
* @returns {{schemaVersion: number, type: 'council-stats', models: Array<object>}}
|
|
88
|
+
*/
|
|
89
|
+
function buildStatsDoc(models) {
|
|
90
|
+
const { COUNCIL_SCHEMA_VERSION } = require('./tally');
|
|
91
|
+
return { schemaVersion: COUNCIL_SCHEMA_VERSION, type: 'council-stats', models };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
module.exports = { buildLedgerRows, appendRun, deriveReliability, buildStatsDoc, LEDGER_FILE, LEDGER_SCHEMA_VERSION };
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// src/council/parse-stage2.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module council/parse-stage2
|
|
6
|
+
* Stage-2 output parsing for the headless council engine (spec §5): the
|
|
7
|
+
* judge's trailing JSON block ({ranking, adjudications}) and the chair's
|
|
8
|
+
* final `VERDICT:` line. Shares last-JSON-block extraction with findings.js.
|
|
9
|
+
* Pure — the ≤2-repair loop lives in run-stages.js; the tri-state
|
|
10
|
+
* (clean|repaired|unstructured) is recorded by the driver.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const { lastJsonBlock } = require('./findings');
|
|
14
|
+
|
|
15
|
+
const JUDGE_VERDICTS = ['agree', 'dispute', 'neutral'];
|
|
16
|
+
const CHAIR_VERDICTS = ['Ship it', 'Fix these first', 'Fundamental rethink'];
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Parse + shape-validate one judge's output.
|
|
20
|
+
* @param {string} text raw judge output (prose + trailing ```json block)
|
|
21
|
+
* @param {{labels: string[], findingIds: string[]}} ctx known labels/ids
|
|
22
|
+
* @returns {{ok: boolean, ranking: Array|null, adjudications: Array|null,
|
|
23
|
+
* errors: Array<{code: string, detail: string}>}}
|
|
24
|
+
*/
|
|
25
|
+
function parseJudgeOutput(text, { labels, findingIds }) {
|
|
26
|
+
const fail = (errors) => ({ ok: false, ranking: null, adjudications: null, errors });
|
|
27
|
+
const body = lastJsonBlock(text || '');
|
|
28
|
+
if (body === null) {
|
|
29
|
+
return fail([{ code: 'NO_FENCED_BLOCK', detail: 'no ```json block found' }]);
|
|
30
|
+
}
|
|
31
|
+
let parsed;
|
|
32
|
+
try { parsed = JSON.parse(body); }
|
|
33
|
+
catch (e) { return fail([{ code: 'NOT_PARSEABLE', detail: e.message }]); }
|
|
34
|
+
|
|
35
|
+
const errors = [];
|
|
36
|
+
const known = new Set(labels);
|
|
37
|
+
const flat = [];
|
|
38
|
+
if (!Array.isArray(parsed.ranking) || parsed.ranking.length === 0) {
|
|
39
|
+
errors.push({ code: 'BAD_RANKING', detail: 'ranking must be a non-empty array of review labels' });
|
|
40
|
+
} else {
|
|
41
|
+
for (const slot of parsed.ranking) {
|
|
42
|
+
for (const label of (Array.isArray(slot) ? slot : [slot])) {
|
|
43
|
+
if (!known.has(label)) {
|
|
44
|
+
errors.push({ code: 'UNKNOWN_LABEL', detail: `unknown review label '${label}'` });
|
|
45
|
+
}
|
|
46
|
+
flat.push(label);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (new Set(flat).size !== flat.length) {
|
|
50
|
+
errors.push({ code: 'DUPLICATE_LABEL', detail: 'a review label appears more than once' });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const knownIds = new Set(findingIds);
|
|
55
|
+
if (!Array.isArray(parsed.adjudications)) {
|
|
56
|
+
errors.push({ code: 'BAD_ADJUDICATIONS', detail: 'adjudications must be an array' });
|
|
57
|
+
} else {
|
|
58
|
+
for (const a of parsed.adjudications) {
|
|
59
|
+
const id = a && a.id;
|
|
60
|
+
if (!knownIds.has(id)) {
|
|
61
|
+
errors.push({ code: 'UNKNOWN_FINDING_ID', detail: `unknown finding id '${id}'` });
|
|
62
|
+
}
|
|
63
|
+
if (!a || !JUDGE_VERDICTS.includes(a.verdict)) {
|
|
64
|
+
errors.push({ code: 'BAD_VERDICT', detail: `bad verdict '${a && a.verdict}' on '${id}'` });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (errors.length) { return fail(errors); }
|
|
70
|
+
return { ok: true, ranking: parsed.ranking, adjudications: parsed.adjudications, errors: [] };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Per-phrase `^<phrase>(?![A-Za-z0-9])` matchers — prefix-anchored, case-sensitive. */
|
|
74
|
+
const CHAIR_VERDICT_PREFIXES = CHAIR_VERDICTS.map(
|
|
75
|
+
(v) => new RegExp('^' + v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?![A-Za-z0-9])')
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Parse the chair's final verdict line. Last matching `VERDICT:` line wins. A
|
|
80
|
+
* line matches when the text after `VERDICT:` equals a canonical phrase, or
|
|
81
|
+
* starts with one followed by a word boundary (trailing rationale, e.g.
|
|
82
|
+
* `VERDICT: Fix these first — <gaps>`). Returns the canonical phrase, not the
|
|
83
|
+
* trailing text.
|
|
84
|
+
* @param {string} text
|
|
85
|
+
* @returns {string|null} one of CHAIR_VERDICTS, or null
|
|
86
|
+
*/
|
|
87
|
+
function parseChairVerdict(text) {
|
|
88
|
+
let found = null;
|
|
89
|
+
for (const line of String(text || '').split('\n')) {
|
|
90
|
+
const m = line.match(/^\s*VERDICT:\s*(.+?)\s*$/);
|
|
91
|
+
if (!m) { continue; }
|
|
92
|
+
const rest = m[1];
|
|
93
|
+
for (let i = 0; i < CHAIR_VERDICTS.length; i++) {
|
|
94
|
+
if (rest === CHAIR_VERDICTS[i] || CHAIR_VERDICT_PREFIXES[i].test(rest)) {
|
|
95
|
+
found = CHAIR_VERDICTS[i];
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return found;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
module.exports = { parseJudgeOutput, parseChairVerdict, CHAIR_VERDICTS, JUDGE_VERDICTS };
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// src/council/run-assemble.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module council/run-assemble
|
|
6
|
+
* Pure assembly + artifact emission for the headless council engine (spec §5):
|
|
7
|
+
* the five-keys tally input (meta pins: claudeInCouncil false, models = bench
|
|
8
|
+
* seats exactly — critic included, chair excluded — runType 'headless'),
|
|
9
|
+
* runStats rows copied verbatim from leg docs, and the run-dir artifact set
|
|
10
|
+
* (tally-input.json, tally.json, verdict.json with overallVerdict, report.html,
|
|
11
|
+
* chair-output.md). Raiser self-votes are INCLUDED in adjudications — exclusion
|
|
12
|
+
* is tally's job (tally.js:95); judged is tally's job (tally.js:110).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const fs = require('fs');
|
|
16
|
+
const path = require('path');
|
|
17
|
+
const { writeFileAtomic } = require('../utils/atomic-write');
|
|
18
|
+
const { buildVerdict, writeVerdictAtomic } = require('./verdict');
|
|
19
|
+
const { buildReport } = require('./report');
|
|
20
|
+
|
|
21
|
+
const CONFORMANCE_RANK = { clean: 0, repaired: 1, unstructured: 2 };
|
|
22
|
+
|
|
23
|
+
/** Worst-wins merge of Stage-1 findings conformance and Stage-2 judge conformance. */
|
|
24
|
+
function worseConformance(a, b) {
|
|
25
|
+
return (CONFORMANCE_RANK[a] || 0) >= (CONFORMANCE_RANK[b] || 0) ? a : b;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* One runStats row from a leg run document. Verbatim copies only — a missing
|
|
30
|
+
* leg doc yields durationMs/usage null (never invent a value). `model` (the
|
|
31
|
+
* council alias) overrides leg.model (the resolved executable id) so ledger
|
|
32
|
+
* rows join meta.models by exact string (ledger.js:20-24).
|
|
33
|
+
*/
|
|
34
|
+
function buildRunStatsEntry({ leg, model, role, wasChair, conformance }) {
|
|
35
|
+
return {
|
|
36
|
+
model: model !== undefined ? model : (leg ? leg.model : null),
|
|
37
|
+
role,
|
|
38
|
+
wasChair: !!wasChair,
|
|
39
|
+
conformance: conformance || 'clean',
|
|
40
|
+
status: leg ? leg.status : 'error',
|
|
41
|
+
durationMs: leg && typeof leg.durationMs === 'number' ? leg.durationMs : null,
|
|
42
|
+
usage: (leg && leg.usage) || null,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Assemble the five-keys tally input (spec §5 / SKILL.md Stage-2 recipe).
|
|
48
|
+
* @param {{runId: string, date: string, bench: string[], chair: string,
|
|
49
|
+
* reviews: Array<{model, role, conformance, leg, globalFindings}>,
|
|
50
|
+
* judgeResults: Array<{judge, ok, order, adjudications}>,
|
|
51
|
+
* chairStats: object|null}} args
|
|
52
|
+
*/
|
|
53
|
+
function buildTallyInput({ runId, date, bench, chair, reviews, judgeResults, chairStats }) {
|
|
54
|
+
const meta = {
|
|
55
|
+
runId, date, runType: 'headless',
|
|
56
|
+
models: bench.slice(), // bench seats exactly: critic included, chair excluded
|
|
57
|
+
chair,
|
|
58
|
+
claudeInCouncil: false, // pinned for headless runs
|
|
59
|
+
};
|
|
60
|
+
const findings = reviews.flatMap(r => r.globalFindings);
|
|
61
|
+
const okJudges = judgeResults.filter(j => j.ok);
|
|
62
|
+
const adjudications = okJudges.flatMap(j =>
|
|
63
|
+
j.adjudications.map(a => ({ findingId: a.id, judge: j.judge, verdict: a.verdict })));
|
|
64
|
+
const rankings = okJudges.map(j => ({ judge: j.judge, order: j.order }));
|
|
65
|
+
const runStats = reviews.map(r => buildRunStatsEntry({
|
|
66
|
+
leg: r.leg, model: r.model, role: r.role, wasChair: false, conformance: r.conformance,
|
|
67
|
+
}));
|
|
68
|
+
if (chairStats) { runStats.push(chairStats); }
|
|
69
|
+
return { meta, findings, adjudications, rankings, runStats };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Persist the assembled input (auditability) and the tally record. */
|
|
73
|
+
function writeTallyFiles({ runDir, tallyInput, record }) {
|
|
74
|
+
writeFileAtomic(path.join(runDir, 'tally-input.json'),
|
|
75
|
+
JSON.stringify(tallyInput, null, 2), { mode: 0o600 });
|
|
76
|
+
writeFileAtomic(path.join(runDir, 'tally.json'),
|
|
77
|
+
JSON.stringify(record, null, 2), { mode: 0o600 });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Undecided verdict + deterministic report. Sets the nullable overallVerdict
|
|
82
|
+
* (council family v2, Plan A) on buildVerdict's output — independent of
|
|
83
|
+
* buildVerdict's own signature.
|
|
84
|
+
* @returns {object} the verdict written to disk
|
|
85
|
+
*/
|
|
86
|
+
function writeVerdictFiles({ runDir, record, overallVerdict, chairText }) {
|
|
87
|
+
const verdict = buildVerdict(record, []);
|
|
88
|
+
verdict.overallVerdict = (overallVerdict === undefined) ? null : overallVerdict;
|
|
89
|
+
writeVerdictAtomic(path.join(runDir, 'verdict.json'), verdict);
|
|
90
|
+
const html = buildReport({ verdict }, { format: 'html' });
|
|
91
|
+
fs.writeFileSync(path.join(runDir, 'report.html'), html, { mode: 0o600 });
|
|
92
|
+
if (chairText) {
|
|
93
|
+
fs.writeFileSync(path.join(runDir, 'chair-output.md'), chairText, { mode: 0o600 });
|
|
94
|
+
}
|
|
95
|
+
return verdict;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
module.exports = {
|
|
99
|
+
buildRunStatsEntry, worseConformance, buildTallyInput, writeTallyFiles, writeVerdictFiles,
|
|
100
|
+
};
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// src/council/run-launch.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module council/run-launch
|
|
6
|
+
* Council-flavored, DI-injected launch wrappers over the fanout transport
|
|
7
|
+
* (spec §5). Every council call — Stage-1 wave, critic/lens solos, Stage-2
|
|
8
|
+
* judge wave, repair re-prompts, chair — goes through runFanout with:
|
|
9
|
+
* - `--agent Plan` default (fixes the skill-vs-engine 'build' default
|
|
10
|
+
* mismatch, fanout.js:184),
|
|
11
|
+
* - `--no-context` (council briefings are self-contained),
|
|
12
|
+
* - quiet mode (the engine owns stdout; wave docs are consumed in-process).
|
|
13
|
+
* Solos are SINGLE-LEG WAVES: one launch primitive gives every call the same
|
|
14
|
+
* leg contract, budget gate, signal abort, and usage accounting.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @param {{fanoutFn?: Function}} [deps] test seam; default = real runFanout
|
|
22
|
+
* @returns {{launchWave: Function, launchSolo: Function}}
|
|
23
|
+
*/
|
|
24
|
+
function createLaunchers(deps = {}) {
|
|
25
|
+
const fanoutFn = deps.fanoutFn || require('../sidecar/fanout').runFanout;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @param {{models: string[], prompt: string, project: string, waveId: string,
|
|
29
|
+
* timeout?: number, gateway?: string, noValidateModel?: boolean, agent?: string}} opts
|
|
30
|
+
* @returns {Promise<{wave: object|null, exitCode: number}>}
|
|
31
|
+
*/
|
|
32
|
+
async function launchWave(opts) {
|
|
33
|
+
fs.mkdirSync(opts.project, { recursive: true });
|
|
34
|
+
const { wave, exitCode } = await fanoutFn({
|
|
35
|
+
models: opts.models.join(','),
|
|
36
|
+
prompt: opts.prompt,
|
|
37
|
+
promptMeta: { source: 'council-engine', file: null, chars: opts.prompt.length },
|
|
38
|
+
waveId: opts.waveId,
|
|
39
|
+
project: opts.project,
|
|
40
|
+
agent: opts.agent || 'Plan',
|
|
41
|
+
timeout: opts.timeout,
|
|
42
|
+
summaryLength: 'verbose',
|
|
43
|
+
includeContext: false,
|
|
44
|
+
gatewayMode: opts.gateway,
|
|
45
|
+
noValidateModel: opts.noValidateModel,
|
|
46
|
+
json: false,
|
|
47
|
+
quiet: true,
|
|
48
|
+
// Spec §6 judge isolation: pin every leg's OpenCode tool-exec cwd to its
|
|
49
|
+
// own session dir (judges' `project` is `<runDir>/_scratch`, so this
|
|
50
|
+
// scopes them there) and strip inherited MCP servers, so a tool-capable
|
|
51
|
+
// judge can't read the de-anonymized review-*.md files or the plaintext
|
|
52
|
+
// labelMap in run.json sitting in the parent run dir.
|
|
53
|
+
directory: opts.project,
|
|
54
|
+
noMcp: true,
|
|
55
|
+
});
|
|
56
|
+
return { wave, exitCode };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* One-model launch (critic/lens legs, repairs, the chair) as a 1-leg wave.
|
|
61
|
+
* @returns {Promise<{wave: object|null, exitCode: number, leg: object|null}>}
|
|
62
|
+
*/
|
|
63
|
+
async function launchSolo(opts) {
|
|
64
|
+
const { wave, exitCode } = await launchWave({ ...opts, models: [opts.model] });
|
|
65
|
+
const leg = (wave && Array.isArray(wave.legs) && wave.legs[0]) || null;
|
|
66
|
+
return { wave, exitCode, leg };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return { launchWave, launchSolo };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Filesystem-safe model name for review-/judge- artifact filenames. */
|
|
73
|
+
function sanitizeName(model) {
|
|
74
|
+
return String(model).replace(/[^a-zA-Z0-9._-]/g, '-');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Write `review-<modelInput>.md` per surviving Stage-1 leg (skill layout).
|
|
79
|
+
* Dead legs and empty summaries are skipped — the caller applies the
|
|
80
|
+
* wave-degrade rules to what remains.
|
|
81
|
+
* @param {string} runDir
|
|
82
|
+
* @param {Array<object>} legs run documents from the wave/solo docs
|
|
83
|
+
* @returns {Array<{model: string, modelInput: string, file: string, text: string, leg: object}>}
|
|
84
|
+
*/
|
|
85
|
+
function materializeReviews(runDir, legs) {
|
|
86
|
+
const out = [];
|
|
87
|
+
for (const leg of legs) {
|
|
88
|
+
if (!leg || leg.status !== 'complete') { continue; }
|
|
89
|
+
const text = leg.summary;
|
|
90
|
+
if (!text || !String(text).trim()) { continue; }
|
|
91
|
+
const modelInput = leg.modelInput || leg.model;
|
|
92
|
+
const file = path.join(runDir, `review-${sanitizeName(modelInput)}.md`);
|
|
93
|
+
fs.writeFileSync(file, text, { mode: 0o600 });
|
|
94
|
+
out.push({ model: leg.model, modelInput, file, text, leg });
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
module.exports = { createLaunchers, materializeReviews, sanitizeName };
|