great-cto 3.25.0 → 3.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "great-cto",
3
3
  "description": "You already have the agent. This is everything around it. great_cto runs Claude Code as a pipeline of 70 specialist agents \u2014 an independent model checks each stage before the next builds on it, spending caps refuse rather than warn, and three decisions stay yours: what gets built, how, and whether it ships.",
4
- "version": "3.25.0",
4
+ "version": "3.26.0",
5
5
  "author": {
6
6
  "name": "Alexander Velikiy",
7
7
  "url": "https://hashnode.com/@Greatcto"
@@ -21,6 +21,9 @@ import { bdCacheInvalidate, checkBeadsAvailable, bdWriteSerialised, bd, bdErr, g
21
21
  import { getMetrics } from './metrics.mjs';
22
22
  import { readVerdicts } from './verdicts.mjs';
23
23
  import { parseAgentBudgets, upsertAgentBudget, removeAgentBudget } from '../../../scripts/lib/agent-budget.mjs';
24
+ import { resolveSecondOpinion, SECOND_OPINION_PROVIDERS } from '../../../scripts/lib/second-opinion.mjs';
25
+ import { detectCodex } from '../../../scripts/lib/codex-exec.mjs';
26
+ import { upsertCapability, capabilitiesFromProjectMd } from '../../../scripts/lib/stack-capabilities.mjs';
24
27
  import { getAgentsFleet, getAgentProfile, retireAgent, restoreAgent, appendDecisionLog, readDecisionsLog } from './fleet.mjs';
25
28
  import { getResume, getShareState, toggleShare } from './share.mjs';
26
29
  import { listSessions, readSession, editedFiles, searchSessions } from './transcripts.mjs';
@@ -1595,6 +1598,123 @@ async function dispatch(req, res, url, cwd) {
1595
1598
 
1596
1599
  // Set or clear one agent's spending cap, by writing PROJECT.md.
1597
1600
  //
1601
+ // GET /api/harnesses — the two harnesses and which one gives the second opinion.
1602
+ //
1603
+ // Claude Code is the host: it runs the pipeline, so it is always "here". Codex
1604
+ // is detected, not assumed — version, login and the model it would run, read
1605
+ // from disk — and reported in three states, because a card that shows an
1606
+ // absent Codex as a quiet toggle in the off position is the same defect as a
1607
+ // pending gate that renders as passed. The evidence section is the tail of
1608
+ // `.great_cto/cross-review.log`: what the second opinion DID, not what it is
1609
+ // set to.
1610
+ if (pathname === '/api/harnesses' && req.method === 'GET') {
1611
+ const c = url.searchParams.get('project') ? resolveProjectCwd(url.searchParams.get('project')) : cwd;
1612
+ const mdPath = path.join(c, '.great_cto', 'PROJECT.md');
1613
+ let projectMd = null;
1614
+ try { projectMd = fs.readFileSync(mdPath, 'utf8'); } catch { projectMd = null; }
1615
+ const codex = detectCodex();
1616
+ const second = resolveSecondOpinion({ projectMd: projectMd ?? '', codex });
1617
+ const declared = projectMd == null ? null : capabilitiesFromProjectMd(projectMd).map.second_opinion;
1618
+
1619
+ // Last 20 review lines, newest first; unparseable lines are counted, not
1620
+ // dropped, so a corrupted log does not read as a quiet one.
1621
+ let evidence = [], unreadable = 0, logState = 'absent';
1622
+ try {
1623
+ const raw = fs.readFileSync(path.join(c, '.great_cto', 'cross-review.log'), 'utf8');
1624
+ logState = 'ok';
1625
+ for (const line of raw.trim().split('\n').filter(Boolean)) {
1626
+ try { evidence.push(JSON.parse(line)); } catch { unreadable += 1; }
1627
+ }
1628
+ evidence = evidence.slice(-20).reverse();
1629
+ } catch (e) { logState = e.code === 'ENOENT' ? 'absent' : 'unreadable'; }
1630
+ const reviewed = evidence.filter((r) => r.state === 'ok');
1631
+ const summary = {
1632
+ runs: evidence.length,
1633
+ reviewed: reviewed.length,
1634
+ skipped: evidence.filter((r) => r.state !== 'ok').length,
1635
+ blocked: reviewed.filter((r) => r.verdict === 'BLOCK').length,
1636
+ unreadable_lines: unreadable,
1637
+ };
1638
+
1639
+ res.writeHead(200, verdictHeaders(c));
1640
+ res.end(JSON.stringify({
1641
+ claude_code: { state: 'host', version: BUILD_VERSION },
1642
+ codex,
1643
+ second_opinion: {
1644
+ ...second,
1645
+ declared: declared ?? { state: projectMd == null ? 'no-project-md' : 'undeclared', tool: null },
1646
+ providers: SECOND_OPINION_PROVIDERS,
1647
+ },
1648
+ evidence: { state: logState, summary, recent: evidence },
1649
+ }));
1650
+ return true;
1651
+ }
1652
+
1653
+ // POST /api/harnesses/second-opinion { provider: codex|openrouter|none|null }
1654
+ //
1655
+ // Writes `capabilities: second_opinion:` into PROJECT.md — a file the operator
1656
+ // owns and git tracks — from a browser, behind the same origin gate as
1657
+ // /api/agent-budgets. `null` removes the key: back to UNDECLARED, which is a
1658
+ // different state from `none` and is offered on purpose.
1659
+ //
1660
+ // The reply carries the previous value, and the resolved state AFTER the
1661
+ // write: choosing codex on a machine without one succeeds as a declaration
1662
+ // and comes back `unavailable`, so the card can say so at the moment of the
1663
+ // click rather than on the next review.
1664
+ if (pathname === '/api/harnesses/second-opinion' && req.method === 'POST') {
1665
+ if (!originAllowed(req)) {
1666
+ res.writeHead(403, { 'Content-Type': 'application/json' });
1667
+ res.end(JSON.stringify({ error: 'origin not allowed' }));
1668
+ return true;
1669
+ }
1670
+ let body = '';
1671
+ req.on('data', (ch) => { body += ch; if (body.length > 1024) req.destroy(); });
1672
+ req.on('end', () => {
1673
+ let parsed;
1674
+ try { parsed = JSON.parse(body || '{}'); }
1675
+ catch (e) {
1676
+ res.writeHead(400, { 'Content-Type': 'application/json' });
1677
+ res.end(JSON.stringify({ error: 'invalid_json', message: String(e.message || e) }));
1678
+ return;
1679
+ }
1680
+ const provider = parsed.provider == null ? null : String(parsed.provider).toLowerCase();
1681
+ if (provider != null && !SECOND_OPINION_PROVIDERS.includes(provider)) {
1682
+ res.writeHead(400, { 'Content-Type': 'application/json' });
1683
+ res.end(JSON.stringify({ error: `provider must be one of ${SECOND_OPINION_PROVIDERS.join(', ')}, or null to undeclare` }));
1684
+ return;
1685
+ }
1686
+ const c = url.searchParams.get('project') ? resolveProjectCwd(url.searchParams.get('project')) : cwd;
1687
+ const mdPath = path.join(c, '.great_cto', 'PROJECT.md');
1688
+ let before;
1689
+ try { before = fs.readFileSync(mdPath, 'utf8'); }
1690
+ catch (e) {
1691
+ res.writeHead(409, { 'Content-Type': 'application/json' });
1692
+ res.end(JSON.stringify({ error: 'no PROJECT.md to write to', detail: String(e.message || e) }));
1693
+ return;
1694
+ }
1695
+ let out;
1696
+ try { out = upsertCapability(before, 'second_opinion', provider); }
1697
+ catch (e) {
1698
+ res.writeHead(400, { 'Content-Type': 'application/json' });
1699
+ res.end(JSON.stringify({ error: String(e.message || e) }));
1700
+ return;
1701
+ }
1702
+ try {
1703
+ const tmp = `${mdPath}.tmp-${process.pid}`;
1704
+ fs.writeFileSync(tmp, out.text);
1705
+ fs.renameSync(tmp, mdPath);
1706
+ } catch (e) {
1707
+ res.writeHead(500, { 'Content-Type': 'application/json' });
1708
+ res.end(JSON.stringify({ error: `could not write PROJECT.md: ${String(e.message || e)}` }));
1709
+ return;
1710
+ }
1711
+ const resolved = resolveSecondOpinion({ projectMd: out.text, codex: detectCodex() });
1712
+ res.writeHead(200, { 'Content-Type': 'application/json' });
1713
+ res.end(JSON.stringify({ ok: true, provider, previous: out.previous, created_block: out.created === true, resolved }));
1714
+ });
1715
+ return true;
1716
+ }
1717
+
1598
1718
  // POST /api/agent-budgets { agent, limit_usd } set / replace
1599
1719
  // POST /api/agent-budgets { agent, remove: true } clear
1600
1720
  //
@@ -1419,6 +1419,20 @@ h1, h2, h3, h4 { font-weight: 600; font-family: var(--sans); }
1419
1419
  }
1420
1420
 
1421
1421
  /* ── change_tier badge ────────────────────────────────────────────────────── */
1422
+ /* `.warn` was used by the Harnesses card before it existed: five spans that
1423
+ said "declared but unavailable" and rendered as ordinary prose. A warning
1424
+ that does not look like a warning is this repository's own defect class,
1425
+ and the css-tokens guard could not see it — it checks var() tokens, not
1426
+ class names. Declared here, and asserted by the harnesses UI test. */
1427
+ .warn { color: var(--status-blocked); }
1428
+ /* `.muted` is used 68 times across this board and was declared nowhere, so
1429
+ every de-emphasised line rendered at full weight — found by the guard added
1430
+ for the Harnesses card, not by the eye. One declaration, the same token the
1431
+ rest of the board de-emphasises with. */
1432
+ .muted { color: var(--text3); }
1433
+ /* Same shape as `.muted`: the `--mono` token exists and is used 115×, the
1434
+ CLASS that applies it did not. */
1435
+ .mono { font-family: var(--mono); }
1422
1436
  .tier-badge { font-size: var(--fs-caption); padding: 1px 8px; border-radius: 999px; margin-left: 6px;
1423
1437
  font-family: var(--mono); white-space: nowrap; }
1424
1438
  .tier-badge:empty { display: none; }
@@ -3174,9 +3188,24 @@ id var(--border);
3174
3188
  <h3>Active pipeline</h3>
3175
3189
  <span class="inbox-count" id="pipeline-status">idle</span>
3176
3190
  <span class="tier-badge" id="tier-badge" title="change_tier for the current working-tree diff — which gates + judge open (ADR-003/004)"></span>
3191
+ <span class="tier-badge" id="harness-badge" title="which harness gives the second opinion in review — from capabilities: second_opinion in PROJECT.md"></span>
3177
3192
  </div>
3178
3193
  <div class="pipeline-track" id="pipeline-track"></div>
3179
3194
  </div>
3195
+
3196
+ <!-- Harnesses: the host that runs the pipeline, and the second harness
3197
+ that reviews beside it. Codex is DETECTED, in three states — an
3198
+ absent Codex must not render as a switch in the off position. The
3199
+ evidence rows are the tail of .great_cto/cross-review.log: what the
3200
+ second opinion did, not what it is set to. -->
3201
+ <div class="inbox-section" id="harnesses-section">
3202
+ <div class="inbox-section-head">
3203
+ <span class="dot dot-blue"></span>
3204
+ <h3>Harnesses</h3>
3205
+ <span class="inbox-count" id="harnesses-status">…</span>
3206
+ </div>
3207
+ <div id="harnesses-body" class="muted" style="font-size:13px;line-height:1.5">loading…</div>
3208
+ </div>
3180
3209
  </div>
3181
3210
  </details>
3182
3211
 
@@ -5106,9 +5135,95 @@ async function refreshPipeline() {
5106
5135
  const d = await api(`/api/pipeline${pqs()}`);
5107
5136
  if (Array.isArray(d)) renderPipeline(d);
5108
5137
  renderTierBadge();
5138
+ renderHarnesses();
5109
5139
  }
5110
5140
 
5111
5141
  // change_tier badge — the gate + judge plan for the current working-tree diff (ADR-003/004).
5142
+ // The Harnesses card. Four states for the second opinion, and the fourth is
5143
+ // the reason the card exists: `unavailable` — declared codex, no codex here —
5144
+ // must never look like `none` or like a switch someone turned off.
5145
+ async function renderHarnesses() {
5146
+ const body = document.getElementById('harnesses-body');
5147
+ const status = document.getElementById('harnesses-status');
5148
+ if (!body) return;
5149
+ let h;
5150
+ try { h = await api(`/api/harnesses${pqs()}`); }
5151
+ catch (e) { status.textContent = 'unreadable'; body.innerHTML = `<b>Could not read the harnesses.</b> ${esc(String(e.message || e))}`; return; }
5152
+
5153
+ const so = h.second_opinion || {};
5154
+ const cx = h.codex || {};
5155
+ const ev = h.evidence || {};
5156
+ const sum = ev.summary || {};
5157
+
5158
+ const codexLine = cx.state === 'available'
5159
+ ? `<b>OpenAI Codex</b> ${esc(cx.version || '')} · logged in (${esc(cx.auth || '?')}) · runs <code>${esc(cx.model || 'default model')}</code>`
5160
+ : cx.state === 'no-auth'
5161
+ ? `<b>OpenAI Codex</b> ${esc(cx.version || '')} · <span class="warn">not logged in</span> — ${esc(cx.why || '')}`
5162
+ : `<b>OpenAI Codex</b> · <span class="warn">not installed</span> — ${esc(cx.why || '')}`;
5163
+
5164
+ const stateLabel = {
5165
+ declared: `on — <b>${esc(so.provider)}</b> reviews in parallel with the Claude reviewer`,
5166
+ none: 'off — the project decided: one model family only',
5167
+ undeclared: 'not declared — <b>not</b> the same as off; nobody has said',
5168
+ unavailable: `<span class="warn">declared <b>${esc(so.provider)}</b>, but unavailable here</span> — ${esc(so.why || '')}`,
5169
+ }[so.state] || esc(so.state || '?');
5170
+
5171
+ const sel = (so.providers || ['codex', 'openrouter', 'none']).map((p) =>
5172
+ `<option value="${p}" ${so.declared && so.declared.tool === p ? 'selected' : ''}>${p}</option>`).join('');
5173
+ const undecl = `<option value="" ${!so.declared || so.declared.state === 'undeclared' ? 'selected' : ''}>(undeclared)</option>`;
5174
+
5175
+ const rows = (ev.recent || []).slice(0, 6).map((r) => {
5176
+ const when = (r.ts || '').slice(0, 16).replace('T', ' ');
5177
+ const what = r.state === 'ok'
5178
+ ? `${r.verdict === 'BLOCK' ? '<b>BLOCK</b>' : 'PASS'} · ${r.findings ?? 0} finding${r.findings === 1 ? '' : 's'}${r.p0 ? ` · ${r.p0} P0` : ''}${r.cost != null ? ` · $${r.cost}` : ' · unpriced'}`
5179
+ : `<span class="muted">skipped (${esc(r.state)})</span>`;
5180
+ return `<div class="mono" style="font-size:12px">${esc(when)} · ${esc(r.provider || '?')}${r.model ? ':' + esc(r.model) : ''} · ${what}</div>`;
5181
+ }).join('');
5182
+ const evidence = ev.state === 'absent'
5183
+ ? '<span class="muted">no cross-model review has run in this project yet</span>'
5184
+ : ev.state === 'unreadable'
5185
+ ? '<span class="warn">the review log is unreadable</span>'
5186
+ : `${sum.reviewed ?? 0} reviewed · ${sum.skipped ?? 0} skipped · ${sum.blocked ?? 0} blocked${sum.unreadable_lines ? ` · <span class="warn">${sum.unreadable_lines} unreadable line(s)</span>` : ''}<div style="margin-top:4px">${rows}</div>`;
5187
+
5188
+ status.textContent = so.state === 'declared' ? `2 · ${so.provider}` : so.state === 'unavailable' ? 'unavailable' : so.state === 'undeclared' ? 'undeclared' : '1';
5189
+ status.className = 'inbox-count' + (so.state === 'unavailable' ? ' warn' : '');
5190
+ // The header badge: two harnesses when a second one actually reviews.
5191
+ // `unavailable` is shown as such — a declared-but-absent Codex in the header
5192
+ // reading "2 harnesses" would be the pending-gate-as-passed defect in a badge.
5193
+ const hb = document.getElementById('harness-badge');
5194
+ if (hb) {
5195
+ hb.textContent = so.state === 'declared' ? `2 harnesses · ${so.provider}`
5196
+ : so.state === 'unavailable' ? `${so.provider} unavailable` : '';
5197
+ hb.className = 'tier-badge';
5198
+ hb.style.color = so.state === 'unavailable' ? 'var(--status-blocked)' : '';
5199
+ hb.style.display = hb.textContent ? '' : 'none';
5200
+ }
5201
+ body.innerHTML = `
5202
+ <div><b>Claude Code</b> · host · runs the pipeline, gates and hooks</div>
5203
+ <div>${codexLine}</div>
5204
+ <div style="margin-top:8px">Second opinion: ${stateLabel}</div>
5205
+ <div style="margin-top:6px;display:flex;gap:8px;align-items:center;flex-wrap:wrap">
5206
+ <label for="so-select" class="muted">set</label>
5207
+ <select id="so-select">${undecl}${sel}</select>
5208
+ <button id="so-apply" class="btn-ghost-sm">apply</button>
5209
+ <span id="so-result" class="muted"></span>
5210
+ </div>
5211
+ <div style="margin-top:8px"><span class="muted">Evidence:</span> ${evidence}</div>`;
5212
+
5213
+ document.getElementById('so-apply').onclick = async () => {
5214
+ const v = document.getElementById('so-select').value;
5215
+ const out = document.getElementById('so-result');
5216
+ out.textContent = 'writing PROJECT.md…';
5217
+ try {
5218
+ const r = await api(`/api/harnesses/second-opinion${pqs()}`, { method: 'POST', body: JSON.stringify({ provider: v || null }) });
5219
+ // Say what it replaced and what it resolved to NOW, so choosing codex on a
5220
+ // machine without one is answered at the click, not on the next review.
5221
+ out.textContent = `was ${r.previous ?? 'undeclared'} → now ${r.provider ?? 'undeclared'} (${r.resolved?.state}${r.resolved?.why ? ': ' + r.resolved.why : ''})`;
5222
+ renderHarnesses();
5223
+ } catch (e) { out.textContent = `failed: ${String(e.message || e)}`; }
5224
+ };
5225
+ }
5226
+
5112
5227
  async function renderTierBadge() {
5113
5228
  const el = document.getElementById('tier-badge');
5114
5229
  if (!el) return;
@@ -0,0 +1,167 @@
1
+ /**
2
+ * codex-exec — the Codex CLI as a subprocess, and whether one is here to run.
3
+ *
4
+ * Moved out of tests/eval/arm-codex.mjs the day Codex became a participant in
5
+ * the pipeline rather than a thing the evals compare against. The parser and
6
+ * runner are unchanged in behaviour; what is new is `detectCodex`, because a
7
+ * second opinion that is silently absent reads exactly like a second opinion
8
+ * that agreed.
9
+ *
10
+ * Three things a caller must be able to tell apart, and none may look like PASS:
11
+ * absent no `codex` on PATH
12
+ * no-auth a binary and no login — `codex exec` returns 400 on every model
13
+ * available version, auth mode and the model it will run, read from disk
14
+ */
15
+ import { spawn, spawnSync } from 'node:child_process';
16
+ import { existsSync, readFileSync } from 'node:fs';
17
+ import { homedir } from 'node:os';
18
+ import { join } from 'node:path';
19
+
20
+ /**
21
+ * `codex exec --json` output → the answer, its cost, and what went wrong.
22
+ *
23
+ * Four states, and the last three are why this is not a one-liner:
24
+ *
25
+ * ok an agent message was produced
26
+ * empty the stream parsed and carried no agent message
27
+ * unreadable nothing in the stream parsed at all
28
+ * (errors) non-fatal problems Codex reported mid-run, always surfaced
29
+ *
30
+ * `empty` returns `text: null`, never `""`. An empty answer graded as an answer
31
+ * scores like a real one. `usage` is null when the turn did not report it — a
32
+ * cost comparison that reads a missing measurement as zero makes one harness
33
+ * look free.
34
+ */
35
+ export function parseCodexStream(raw) {
36
+ const messages = [];
37
+ const errors = [];
38
+ let usage = null;
39
+ let parsedAny = false;
40
+
41
+ for (const line of String(raw ?? '').split('\n')) {
42
+ const s = line.trim();
43
+ // The CLI interleaves human-readable lines with the JSON stream, so a line
44
+ // that does not parse is noise to step over, not a failure.
45
+ if (!s.startsWith('{')) continue;
46
+ let ev;
47
+ try { ev = JSON.parse(s); } catch { continue; }
48
+ parsedAny = true;
49
+
50
+ if (ev.type === 'turn.completed' && ev.usage) usage = ev.usage;
51
+ const item = ev.item;
52
+ if (!item) continue;
53
+ if (item.type === 'agent_message' && typeof item.text === 'string') messages.push(item.text);
54
+ // Codex reports recoverable problems as error items: a rejected plugin
55
+ // config, a truncated skill budget. A verdict built over a degraded run is
56
+ // a verdict about the wrong thing, so these always reach the caller.
57
+ else if (item.type === 'error' && item.message) errors.push(String(item.message));
58
+ }
59
+
60
+ if (!parsedAny) return { state: 'unreadable', text: null, usage: null, errors };
61
+ if (!messages.length) return { state: 'empty', text: null, usage, errors };
62
+ return { state: 'ok', text: messages.join('\n'), usage, errors };
63
+ }
64
+
65
+ /**
66
+ * Run one prompt through the Codex CLI and parse the result.
67
+ *
68
+ * The prompt goes on stdin: passing it as an argument alongside `-c` overrides
69
+ * made the CLI wait on stdin instead, which looks exactly like a hung model.
70
+ *
71
+ * `sandbox` defaults to read-only: a reviewer that can write is not a reviewer.
72
+ * `ephemeral` keeps the review out of the user's Codex session history.
73
+ *
74
+ * @returns the shape of `parseCodexStream`, plus `{ code, model }`
75
+ */
76
+ export function runCodexExec({
77
+ prompt, cwd, timeoutMs = 300000, bin = 'codex', model = null,
78
+ sandbox = 'read-only', ephemeral = true, extraArgs = [],
79
+ }) {
80
+ return new Promise((resolve) => {
81
+ const args = ['exec', '--json', '--skip-git-repo-check'];
82
+ if (sandbox) args.push('-s', sandbox);
83
+ if (ephemeral) args.push('--ephemeral');
84
+ if (model) args.push('-m', model);
85
+ if (cwd) args.push('-C', cwd);
86
+ args.push(...extraArgs, '-');
87
+
88
+ const proc = spawn(bin, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env } });
89
+ let out = '';
90
+ let err = '';
91
+ const timer = setTimeout(() => { try { proc.kill('SIGKILL'); } catch { /* gone */ } }, timeoutMs);
92
+
93
+ proc.stdout.on('data', (b) => { out += String(b); });
94
+ proc.stderr.on('data', (b) => { err += String(b); });
95
+ proc.on('error', (e) => {
96
+ clearTimeout(timer);
97
+ resolve({ state: 'unreadable', text: null, usage: null, errors: [String(e.message || e)], code: null, model });
98
+ });
99
+ proc.on('close', (code) => {
100
+ clearTimeout(timer);
101
+ const parsed = parseCodexStream(out);
102
+ // stderr is kept even on success: Codex writes warnings there that change
103
+ // how a result should be read.
104
+ if (err.trim()) parsed.errors.push(err.trim().slice(0, 500));
105
+ resolve({ ...parsed, code, model });
106
+ });
107
+
108
+ proc.stdin.write(prompt);
109
+ proc.stdin.end();
110
+ });
111
+ }
112
+
113
+ /** Kept under the old name for the eval arm. */
114
+ export const runCodexArm = (opts) => runCodexExec({ ...opts, sandbox: opts.sandbox ?? null, ephemeral: opts.ephemeral ?? false });
115
+
116
+ /**
117
+ * What a Codex install on this machine is, from three readings — pure, so the
118
+ * three states can be tested without a Codex.
119
+ *
120
+ * @param {{versionOut:string|null, authJson:string|null, configToml:string|null}} r
121
+ * `null` for a reading that could not be taken (binary absent, file missing).
122
+ * @returns {{state:'absent'|'no-auth'|'available', version:string|null,
123
+ * auth:string|null, model:string|null, why:string}}
124
+ */
125
+ export function codexStatusFrom({ versionOut, authJson, configToml }) {
126
+ if (versionOut == null) {
127
+ return { state: 'absent', version: null, auth: null, model: null, why: 'codex is not on PATH — npm i -g @openai/codex' };
128
+ }
129
+ const version = (String(versionOut).match(/(\d+\.\d+\.\d+)/) || [])[1] ?? null;
130
+
131
+ let auth = null;
132
+ if (authJson != null) {
133
+ try { auth = JSON.parse(authJson)?.auth_mode ?? (JSON.parse(authJson)?.tokens ? 'chatgpt' : null); }
134
+ catch { auth = null; }
135
+ }
136
+ // config.toml may name the model on its own line; a `[profiles.x]` table
137
+ // below can name others, so only the first top-level `model =` counts.
138
+ let model = null;
139
+ if (configToml != null) {
140
+ const top = String(configToml).split(/^\[/m)[0];
141
+ model = (top.match(/^\s*model\s*=\s*"([^"]+)"/m) || [])[1] ?? null;
142
+ }
143
+
144
+ if (!auth) {
145
+ return { state: 'no-auth', version, auth: null, model, why: 'codex is installed but not logged in — run `codex login`' };
146
+ }
147
+ return { state: 'available', version, auth, model, why: '' };
148
+ }
149
+
150
+ /** The three readings, taken. */
151
+ // `GREAT_CTO_CODEX_BIN` is the one lever for all three consumers — reviewer,
152
+ // verifier, board — so a test (or an operator with two Codex installs) points
153
+ // them at the same binary, and so an absent Codex can be simulated without
154
+ // stripping PATH of the node that runs the server.
155
+ export function detectCodex({ bin = process.env.GREAT_CTO_CODEX_BIN || 'codex', home = homedir() } = {}) {
156
+ let versionOut = null;
157
+ try {
158
+ const r = spawnSync(bin, ['--version'], { encoding: 'utf8', timeout: 5000 });
159
+ if (r.status === 0) versionOut = String(r.stdout || r.stderr || '');
160
+ } catch { /* absent */ }
161
+ const read = (p) => { try { return existsSync(p) ? readFileSync(p, 'utf8') : null; } catch { return null; } };
162
+ return codexStatusFrom({
163
+ versionOut,
164
+ authJson: read(join(home, '.codex', 'auth.json')),
165
+ configToml: read(join(home, '.codex', 'config.toml')),
166
+ });
167
+ }
@@ -0,0 +1,297 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * dag-metric — score a judgement by walking a graph of narrow questions, instead
4
+ * of asking a model for a number.
5
+ *
6
+ * Why this exists
7
+ * ---------------
8
+ * Our eval judge returns a rate per scenario, and the recorded history says what
9
+ * that costs: 14 of 18 results below their own threshold, one eval at 0.44
10
+ * against a 1.00 bar, and the runner's own header notes ±0.09 of variance that
11
+ * more samples do not remove. That variance is inherent to asking "score this
12
+ * 0–1" — the number is sampled from a distribution, not derived from anything.
13
+ *
14
+ * The idea is DeepEval's, and it is the one thing worth taking from it: reduce
15
+ * the model's job to questions it can answer the same way twice ("does this
16
+ * finding cite a file:line?"), and COMPUTE the score from the path those answers
17
+ * take. The model still judges; it no longer does arithmetic.
18
+ *
19
+ * Two properties follow, and both matter more than the score:
20
+ * - the same answers always give the same score
21
+ * - the verdict is a path you can print, so a disagreement is about a specific
22
+ * question rather than about a number nobody can locate
23
+ *
24
+ * Shape (plain JSON, so a DAG is data an agent can write):
25
+ * { root, nodes: { id: { question, edges: { <answer>: <nextId> } } },
26
+ * leaves: { id: { score: 0..1, reason } } }
27
+ *
28
+ * CLI:
29
+ * node scripts/lib/dag-metric.mjs <dag.json> --answers '{"n1":"yes"}'
30
+ * node scripts/lib/dag-metric.mjs <dag.json> --validate
31
+ * node scripts/lib/dag-metric.mjs <dag.json> --next --answers '{...}'
32
+ */
33
+
34
+ import { createHash } from 'node:crypto';
35
+
36
+ const isLeafScore = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 1;
37
+
38
+ /**
39
+ * A 12-hex-char fingerprint over the DAG's scoring structure — `root`, each
40
+ * node's `question` and `edges`, each leaf's `score` — with every level's keys
41
+ * sorted so key order in the source file cannot move the hash.
42
+ *
43
+ * Deliberately excludes `note` (top-level) and each leaf's `reason`: those are
44
+ * documentation, not the metric. Two runs scored by the same questions and the
45
+ * same edges must fingerprint identically even if someone rewrote a comment —
46
+ * and two runs whose questions or edges differ must not, because that is
47
+ * exactly the case (ARCH-judge-provenance §1/§3) that makes their rates
48
+ * incomparable.
49
+ */
50
+ export function dagFingerprint(dag) {
51
+ const sortKeys = (o) => Object.fromEntries(Object.entries(o || {}).sort(([a], [b]) => (a < b ? -1 : 1)));
52
+ const scoring = {
53
+ root: dag && dag.root,
54
+ nodes: sortKeys(Object.fromEntries(Object.entries((dag && dag.nodes) || {})
55
+ // `n?.` rather than `n.`: a null entry inside `nodes` threw where an
56
+ // absent `nodes` did not, so the function was safe against the malformed
57
+ // shapes anyone thought to try and unsafe against one nobody did. This
58
+ // runs inside an eval that has already been paid for.
59
+ .map(([id, n]) => [id, { question: n?.question, edges: sortKeys(n?.edges) }]))),
60
+ leaves: sortKeys(Object.fromEntries(Object.entries((dag && dag.leaves) || {})
61
+ .map(([id, l]) => [id, { score: l?.score }]))),
62
+ };
63
+ return createHash('sha256').update(JSON.stringify(scoring)).digest('hex').slice(0, 12);
64
+ }
65
+
66
+ /**
67
+ * Check the graph before anyone judges anything: a lost branch or a cycle is a
68
+ * bug in the metric, and finding it mid-run means a wasted judging pass.
69
+ *
70
+ * @returns {{errors: string[], warnings: string[]}}
71
+ */
72
+ export function validateDag(dag) {
73
+ const errors = [];
74
+ const warnings = [];
75
+ const nodes = (dag && dag.nodes) || {};
76
+ const leaves = (dag && dag.leaves) || {};
77
+ const root = dag && dag.root;
78
+
79
+ if (!root) errors.push('no `root`');
80
+ if (!Object.keys(nodes).length) errors.push('no `nodes`');
81
+ if (!Object.keys(leaves).length) errors.push('no `leaves`');
82
+ if (root && !nodes[root] && !leaves[root]) errors.push(`root '${root}' is neither a node nor a leaf`);
83
+
84
+ for (const [id, leaf] of Object.entries(leaves)) {
85
+ if (!leaf || !('score' in leaf)) { errors.push(`leaf '${id}' has no score`); continue; }
86
+ if (!isLeafScore(leaf.score)) errors.push(`leaf '${id}' score must be a number in 0..1`);
87
+ }
88
+
89
+ for (const [id, node] of Object.entries(nodes)) {
90
+ if (!node || !node.question) errors.push(`node '${id}' has no question`);
91
+ const edges = (node && node.edges) || {};
92
+ if (!Object.keys(edges).length) errors.push(`node '${id}' has no edges`);
93
+ for (const [answer, target] of Object.entries(edges)) {
94
+ if (!nodes[target] && !leaves[target]) {
95
+ errors.push(`node '${id}' edge '${answer}' points at '${target}', which does not exist`);
96
+ }
97
+ }
98
+ }
99
+
100
+ // Cycles: a question graph that loops can never reach a score.
101
+ const state = new Map(); // id → 'open' | 'done'
102
+ const walk = (id, trail) => {
103
+ if (leaves[id]) return;
104
+ if (state.get(id) === 'open') { errors.push(`cycle: ${[...trail, id].join(' → ')}`); return; }
105
+ if (state.get(id) === 'done' || !nodes[id]) return;
106
+ state.set(id, 'open');
107
+ for (const target of Object.values(nodes[id].edges || {})) walk(target, [...trail, id]);
108
+ state.set(id, 'done');
109
+ };
110
+ if (root) walk(root, []);
111
+
112
+ // Reachability: an orphaned leaf usually means a branch was dropped in an edit.
113
+ const seen = new Set();
114
+ const reach = (id) => {
115
+ if (!id || seen.has(id)) return;
116
+ seen.add(id);
117
+ for (const target of Object.values((nodes[id] || {}).edges || {})) reach(target);
118
+ };
119
+ if (root && !errors.some((e) => e.startsWith('cycle'))) reach(root);
120
+ for (const id of Object.keys(leaves)) if (!seen.has(id)) warnings.push(`leaf '${id}' is unreachable`);
121
+ for (const id of Object.keys(nodes)) if (!seen.has(id)) warnings.push(`node '${id}' is unreachable`);
122
+
123
+ return { errors, warnings };
124
+ }
125
+
126
+ /**
127
+ * Walk the graph with the answers given.
128
+ *
129
+ * A missing answer returns `score: null` and names the question. It never
130
+ * defaults, because a default is exactly the invented number this module exists
131
+ * to remove.
132
+ */
133
+ export function evaluateDag(dag, answers = {}) {
134
+ const { errors } = validateDag(dag);
135
+ if (errors.length) return { score: null, path: [], leaf: null, error: errors[0] };
136
+
137
+ const { nodes, leaves, root } = dag;
138
+ const path = [];
139
+ const steps = []; // {id, question, answer} — the audit trail
140
+ let id = root;
141
+ const guard = Object.keys(nodes).length + 1; // validation rules out cycles; this is belt
142
+
143
+ for (let step = 0; step <= guard; step++) {
144
+ if (leaves[id]) {
145
+ path.push(id);
146
+ return { score: leaves[id].score, leaf: id, reason: leaves[id].reason || '', path, steps };
147
+ }
148
+ const node = nodes[id];
149
+ path.push(id);
150
+ if (!(id in answers)) {
151
+ return { score: null, path, leaf: null, steps, pending: id, question: node.question };
152
+ }
153
+ const answer = answers[id];
154
+ const next = node.edges[answer];
155
+ if (!next) {
156
+ return {
157
+ score: null, path, leaf: null, steps,
158
+ error: `node '${id}' has no edge for answer '${answer}' (expected: ${Object.keys(node.edges).join(', ')})`,
159
+ };
160
+ }
161
+ steps.push({ id, question: node.question, answer });
162
+ id = next;
163
+ }
164
+ return { score: null, path, leaf: null, steps, error: 'walk did not terminate' };
165
+ }
166
+
167
+ /** The next question to put to a judge, with the answers it may give. */
168
+ export function pendingQuestion(dag, answers = {}) {
169
+ const r = evaluateDag(dag, answers);
170
+ if (!r.pending) return null;
171
+ return { id: r.pending, question: r.question, answers: Object.keys(dag.nodes[r.pending].edges) };
172
+ }
173
+
174
+ /** The verdict a human reads: every question, its answer, and where the score came from. */
175
+ export function explain(result, dag) {
176
+ if (!result) return 'no result';
177
+ if (result.error) return `unscored — ${result.error}`;
178
+
179
+ const lines = (result.steps || []).map((s) => ` ${s.question}\n ${s.answer}`);
180
+
181
+ if (result.score === null) {
182
+ const q = result.question ? `\n ${result.question}\n (unanswered)` : '';
183
+ return [`unscored — pending question: ${result.pending || 'unknown'}`, ...lines].join('\n') + q;
184
+ }
185
+ if (result.leaf) lines.push(` → ${result.reason || result.leaf}`);
186
+ return [`score: ${result.score}`, ...lines].join('\n');
187
+ }
188
+
189
+ /**
190
+ * The prompt for one node. The judge is given the allowed answers and told to
191
+ * reply with nothing else — the whole point of the graph is that its job is a
192
+ * choice from a closed set, not a number it has to invent.
193
+ */
194
+ export function questionPrompt(question, allowed, context = {}) {
195
+ const list = allowed.join(' or ');
196
+ const system =
197
+ 'You are grading one narrow factual question about an agent response. ' +
198
+ `Answer with exactly one word: ${list}. ` +
199
+ 'No explanation, no punctuation, no hedging. If the response genuinely does not ' +
200
+ `settle the question, answer with the option that assumes the agent did NOT do it (${allowed[allowed.length - 1]}).`;
201
+ const user = [
202
+ context.scenario ? `Scenario: ${context.scenario}` : '',
203
+ context.test ? `Test case: ${context.test}` : '',
204
+ context.expected ? `Expected behaviour: ${context.expected}` : '',
205
+ context.actorResponse ? `Agent response:\n${context.actorResponse}` : '',
206
+ '',
207
+ `Question: ${question}`,
208
+ `Answer (${list}):`,
209
+ ].filter(Boolean).join('\n\n');
210
+ return { system, user };
211
+ }
212
+
213
+ /**
214
+ * Map a judge reply onto one of the allowed answers, or null.
215
+ *
216
+ * Null matters: an unparseable reply must stop the walk rather than pick a side,
217
+ * because a guessed answer is a guessed score wearing a decision tree.
218
+ */
219
+ export function parseAnswer(reply, allowed) {
220
+ const t = String(reply ?? '').trim().toLowerCase();
221
+ if (!t) return null;
222
+ // Ambiguity is checked BEFORE any shortcut: a reply that echoes the option
223
+ // list ("yes or no") starts with a valid answer, and taking the first token
224
+ // would turn the judge's non-answer into a decision.
225
+ // Delimit by "not a word character" on either side rather than \b: a label is
226
+ // whatever the graph author wrote, and \b needs a word character to sit against,
227
+ // so `yes(1)` or `p0.` could never match their own exact reply.
228
+ const hits = allowed.filter((a) => {
229
+ const esc = a.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
230
+ return new RegExp(`(^|[^\\w-])${esc}([^\\w-]|$)`).test(t);
231
+ });
232
+ return hits.length === 1 ? hits[0] : null;
233
+ }
234
+
235
+ /**
236
+ * Score one case by walking the graph, asking `ask(question, allowed)` for each
237
+ * node on the path. `ask` returns a string reply; anything unparseable ends the
238
+ * walk unscored rather than guessing.
239
+ *
240
+ * Kept here rather than in the runner so it is testable with a stub `ask`.
241
+ */
242
+ export async function judgeWithDag(dag, ask, context = {}) {
243
+ const answers = {};
244
+ const asked = [];
245
+ for (let i = 0; i <= Object.keys(dag.nodes || {}).length; i++) {
246
+ const q = pendingQuestion(dag, answers);
247
+ if (!q) break;
248
+ const reply = await ask(q.question, q.answers, context);
249
+ const answer = parseAnswer(reply, q.answers);
250
+ asked.push({ id: q.id, question: q.question, reply, answer });
251
+ if (answer === null) {
252
+ return { score: null, answers, asked, error: `judge gave no usable answer for '${q.id}': ${String(reply).slice(0, 60)}` };
253
+ }
254
+ answers[q.id] = answer;
255
+ }
256
+ return { ...evaluateDag(dag, answers), answers, asked };
257
+ }
258
+
259
+ // ── CLI ─────────────────────────────────────────────────────────────────────
260
+
261
+ async function main(argv) {
262
+ const { readFileSync } = await import('node:fs');
263
+ const file = argv.find((a) => !a.startsWith('--'));
264
+ if (!file) {
265
+ console.error('usage: dag-metric.mjs <dag.json> [--validate] [--next] [--answers \'{"id":"yes"}\']');
266
+ return 2;
267
+ }
268
+ let dag;
269
+ try { dag = JSON.parse(readFileSync(file, 'utf8')); }
270
+ catch (e) { console.error(`cannot read ${file}: ${e.message}`); return 2; }
271
+
272
+ const i = argv.indexOf('--answers');
273
+ let answers = {};
274
+ if (i >= 0) {
275
+ try { answers = JSON.parse(argv[i + 1]); }
276
+ catch { console.error('--answers must be JSON'); return 2; }
277
+ }
278
+
279
+ const { errors, warnings } = validateDag(dag);
280
+ for (const w of warnings) console.error(`warn: ${w}`);
281
+ if (errors.length) { for (const e of errors) console.error(`error: ${e}`); return 1; }
282
+ if (argv.includes('--validate')) { console.log(`ok: ${Object.keys(dag.nodes).length} nodes, ${Object.keys(dag.leaves).length} leaves`); return 0; }
283
+
284
+ if (argv.includes('--next')) {
285
+ const q = pendingQuestion(dag, answers);
286
+ console.log(q ? JSON.stringify(q, null, 2) : 'null');
287
+ return 0;
288
+ }
289
+
290
+ const result = evaluateDag(dag, answers);
291
+ console.log(explain(result, dag));
292
+ return result.score === null ? 1 : 0;
293
+ }
294
+
295
+ if (import.meta.url === `file://${process.argv[1]}`) {
296
+ main(process.argv.slice(2)).then((c) => { process.exitCode = c; });
297
+ }
@@ -0,0 +1,325 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * second-opinion — two judges, one graph, and only the disagreement is a signal.
4
+ *
5
+ * Why this exists
6
+ * ---------------
7
+ * `mcp__great_cto_llm_router__ask_kimi` has been in the `tools:` list of
8
+ * nineteen agents and invoked by none of them. The capability was declared and
9
+ * never exercised — the same shape as the deny list that ran on every write and
10
+ * protected nothing.
11
+ *
12
+ * The obvious use, a second model reviewing the first one's report, is the wrong
13
+ * one. A reviewer reading a report judges whether it reads plausibly, and a
14
+ * confident wrong finding is exactly what passes that test. Worse, two models
15
+ * agreeing is weak evidence: they are trained on overlapping data and fail in
16
+ * correlated ways. "Both said yes" is close to no information.
17
+ *
18
+ * What carries information is where they DIVERGE. So this does not ask two
19
+ * models to grade prose. It walks both through the same DAG of closed questions
20
+ * (scripts/lib/dag-metric.mjs) and reports the nodes where their answers differ.
21
+ * A divergence names a specific question — "does this finding cite a file:line?"
22
+ * — which is something a human can settle in seconds, unlike "is this report
23
+ * good?".
24
+ *
25
+ * The reporting rule is deliberate and the opposite of the usual one:
26
+ *
27
+ * - agreement is NOT reported as confidence. It is reported as "nothing to
28
+ * look at here", which is a weaker claim and the honest one.
29
+ * - divergence IS the output. It is where to dig.
30
+ * - one judge abstaining is neither. An unparseable answer is a third state,
31
+ * not a vote, and calling it agreement would let a broken judge confirm
32
+ * anything.
33
+ */
34
+
35
+ import { pendingQuestion, evaluateDag, parseAnswer } from './dag-metric.mjs';
36
+
37
+ export const DIVERGENCE = Object.freeze({
38
+ AGREE: 'agree',
39
+ DIVERGE: 'diverge',
40
+ ABSTAINED: 'abstained',
41
+ });
42
+
43
+ /**
44
+ * Walk one judge through the graph, collecting its answer at every node it
45
+ * reaches. Unlike judgeWithDag this does not stop at the first unusable reply —
46
+ * an abstention on one node still leaves the others worth comparing.
47
+ *
48
+ * @param {object} dag
49
+ * @param {(q:string, allowed:string[], ctx:object)=>Promise<string>} ask
50
+ * @returns {Promise<{answers: object, asked: Array, result: object}>}
51
+ */
52
+ export async function walk(dag, ask, context = {}) {
53
+ const answers = {};
54
+ const asked = [];
55
+ const limit = Object.keys(dag.nodes || {}).length + 1;
56
+ for (let i = 0; i <= limit; i++) {
57
+ const q = pendingQuestion(dag, answers);
58
+ if (!q) break;
59
+ const reply = await ask(q.question, q.answers, context);
60
+ const answer = parseAnswer(reply, q.answers);
61
+ asked.push({ id: q.id, question: q.question, reply, answer });
62
+ // A node nobody could answer ends this judge's walk — the graph cannot
63
+ // continue without it — but what was already answered still compares.
64
+ if (answer === null) break;
65
+ answers[q.id] = answer;
66
+ }
67
+ return { answers, asked, result: evaluateDag(dag, answers) };
68
+ }
69
+
70
+ /**
71
+ * Compare two walks.
72
+ *
73
+ * @returns {{
74
+ * diverged: Array<{id, question, a, b}>,
75
+ * abstained: Array<{id, question, by: 'a'|'b'|'both'}>,
76
+ * agreed: string[],
77
+ * scores: {a: number|null, b: number|null},
78
+ * verdict: 'agree'|'diverge'|'abstained',
79
+ * summary: string,
80
+ * }}
81
+ */
82
+ export function compare(walkA, walkB) {
83
+ const byId = (w) => new Map(w.asked.map((s) => [s.id, s]));
84
+ const A = byId(walkA);
85
+ const B = byId(walkB);
86
+
87
+ const diverged = [];
88
+ const abstained = [];
89
+ const agreed = [];
90
+
91
+ for (const id of new Set([...A.keys(), ...B.keys()])) {
92
+ const a = A.get(id);
93
+ const b = B.get(id);
94
+ // A node only one judge reached is not a disagreement — the walks parted
95
+ // earlier, and the node that parted them is already recorded.
96
+ if (!a || !b) continue;
97
+ if (a.answer === null || b.answer === null) {
98
+ abstained.push({
99
+ id, question: a.question,
100
+ by: a.answer === null && b.answer === null ? 'both' : (a.answer === null ? 'a' : 'b'),
101
+ });
102
+ continue;
103
+ }
104
+ if (a.answer !== b.answer) diverged.push({ id, question: a.question, a: a.answer, b: b.answer });
105
+ else agreed.push(id);
106
+ }
107
+
108
+ const scores = { a: walkA.result?.score ?? null, b: walkB.result?.score ?? null };
109
+
110
+ let verdict = DIVERGENCE.AGREE;
111
+ if (diverged.length) verdict = DIVERGENCE.DIVERGE;
112
+ else if (abstained.length) verdict = DIVERGENCE.ABSTAINED;
113
+
114
+ // The wording matters. Agreement is not evidence of correctness — two models
115
+ // trained on overlapping data fail together — so it is reported as an absence
116
+ // of signal, never as confidence.
117
+ const summary = diverged.length
118
+ ? `${diverged.length} question(s) answered differently — that is where to look`
119
+ : abstained.length
120
+ ? `${abstained.length} question(s) one judge could not answer — no comparison possible there`
121
+ : `no divergence across ${agreed.length} question(s); this is weak evidence, not confirmation`;
122
+
123
+ return { diverged, abstained, agreed, scores, verdict, summary };
124
+ }
125
+
126
+ /**
127
+ * Run both judges over the same graph and compare.
128
+ *
129
+ * The two `ask` functions must be independent — a second opinion that saw the
130
+ * first one's answer is not a second opinion.
131
+ */
132
+ export async function secondOpinion(dag, askA, askB, context = {}) {
133
+ const [a, b] = await Promise.all([walk(dag, askA, context), walk(dag, askB, context)]);
134
+ return { ...compare(a, b), walks: { a, b } };
135
+ }
136
+
137
+ /** Human-readable report. Divergence first — it is the only actionable part. */
138
+ export function explainComparison(cmp) {
139
+ const lines = [cmp.summary];
140
+ for (const d of cmp.diverged) {
141
+ lines.push('', ` ${d.question}`, ` judge A: ${d.a}`, ` judge B: ${d.b}`);
142
+ }
143
+ for (const s of cmp.abstained) {
144
+ lines.push('', ` ${s.question}`, ` unanswered by: ${s.by}`);
145
+ }
146
+ if (cmp.scores.a !== cmp.scores.b) {
147
+ lines.push('', ` scores differ: ${cmp.scores.a} vs ${cmp.scores.b}`);
148
+ }
149
+ return lines.join('\n');
150
+ }
151
+
152
+ // ── the second judge ────────────────────────────────────────────────────────
153
+
154
+ /**
155
+ * An `ask` backed by the llm-router MCP server (Kimi by default).
156
+ *
157
+ * Spawned per question on purpose: the server is stdio JSON-RPC and a fresh
158
+ * process per question guarantees the second judge cannot see its own earlier
159
+ * answers accumulate into a context that biases the next one. It is slower and
160
+ * that is the correct trade for an independence claim.
161
+ */
162
+ export function routerAsk(serverPath, { timeoutMs = 60_000, model = null } = {}) {
163
+ return async (question, allowed) => {
164
+ const { spawn } = await import('node:child_process');
165
+ return new Promise((resolve) => {
166
+ // The model is passed through the spawned server's environment, which is
167
+ // where it reads it from. Without this every "second opinion" was the same
168
+ // model answering twice — three samples of one judge are correlated, and
169
+ // calling that a second opinion is the confidence-by-repetition this
170
+ // module was written to avoid.
171
+ const env = model ? { ...process.env, GREAT_CTO_ROUTER_MODEL: model } : process.env;
172
+ const p = spawn('python3', [serverPath], { stdio: ['pipe', 'pipe', 'ignore'], env });
173
+ let out = '';
174
+ const done = (v) => { try { p.kill(); } catch {} resolve(v); };
175
+ const timer = setTimeout(() => done(''), timeoutMs);
176
+ p.stdout.on('data', (d) => { out += d; });
177
+ p.on('error', () => { clearTimeout(timer); done(''); });
178
+ p.on('close', () => {
179
+ clearTimeout(timer);
180
+ for (const line of out.split('\n')) {
181
+ if (!line.trim().startsWith('{')) continue;
182
+ try {
183
+ const d = JSON.parse(line);
184
+ if (d.id === 2 && d.result && d.result.content) return done(d.result.content[0].text);
185
+ } catch { /* not our line */ }
186
+ }
187
+ done('');
188
+ });
189
+ const task = `${question}\nAnswer with exactly one word from: ${allowed.join(', ')}. No explanation.`;
190
+ p.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }) + '\n');
191
+ p.stdin.write(JSON.stringify({
192
+ jsonrpc: '2.0', id: 2, method: 'tools/call',
193
+ params: { name: 'ask_kimi', arguments: { task } },
194
+ }) + '\n');
195
+ p.stdin.end();
196
+ });
197
+ };
198
+ }
199
+
200
+ // ── CLI ─────────────────────────────────────────────────────────────────────
201
+ //
202
+ // node scripts/lib/second-opinion.mjs <dag.json> --answers '{"node-id":"yes"}'
203
+ //
204
+ // You supply YOUR answers; the router answers the same questions independently
205
+ // and the divergence is printed. Exit 1 when the judges diverge — not because
206
+ // divergence is a failure, but because it is the case a human should see.
207
+
208
+ async function main(argv) {
209
+ const { readFileSync, existsSync } = await import('node:fs');
210
+ const { join, dirname } = await import('node:path');
211
+ const { fileURLToPath } = await import('node:url');
212
+
213
+ const file = argv.find((a) => !a.startsWith('--'));
214
+ const i = argv.indexOf('--answers');
215
+ if (!file || i === -1) {
216
+ console.error('usage: second-opinion.mjs <dag.json> --answers \'{"node-id":"yes"}\' [--server <path>] [--json]');
217
+ return 2;
218
+ }
219
+
220
+ let dag, mine;
221
+ try { dag = JSON.parse(readFileSync(file, 'utf8')); }
222
+ catch (e) { console.error(`cannot read ${file}: ${e.message}`); return 2; }
223
+ try { mine = JSON.parse(argv[i + 1]); }
224
+ catch { console.error('--answers must be JSON'); return 2; }
225
+
226
+ const si = argv.indexOf('--server');
227
+ const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
228
+ const server = si >= 0 ? argv[si + 1] : join(root, 'mcp-servers', 'llm-router', 'server.py');
229
+ if (!existsSync(server)) { console.error(`router not found at ${server}`); return 2; }
230
+
231
+ const cmp = await secondOpinion(dag, async (q, allowed) => {
232
+ const id = Object.keys(dag.nodes).find((k) => dag.nodes[k].question === q);
233
+ // An answer you did not give is an abstention, not a default. Filling it in
234
+ // would manufacture the agreement this tool exists to avoid claiming.
235
+ return id in mine ? String(mine[id]) : '';
236
+ }, routerAsk(server));
237
+
238
+ if (argv.includes('--json')) console.log(JSON.stringify(cmp, null, 2));
239
+ else console.log(explainComparison(cmp));
240
+ return cmp.diverged.length ? 1 : 0;
241
+ }
242
+
243
+ if (import.meta.url === `file://${process.argv[1]}`) {
244
+ main(process.argv.slice(2)).then((c) => { process.exitCode = c; });
245
+ }
246
+
247
+ // ── Which harness gives the second opinion ───────────────────────────────────
248
+ //
249
+ // Until 2026-09-05 the second opinion was whatever OpenRouter model the
250
+ // environment named, and the environment named it in three different variables.
251
+ // Codex is now a participant: `codex exec` in a read-only sandbox, authenticated
252
+ // by the user's ChatGPT login, a genuinely different model family with no API
253
+ // key to leak. Which one runs is a project decision, declared once in
254
+ // PROJECT.md's `capabilities:` block and read here — by the reviewer, by the
255
+ // verifier, and by the board, so all three agree.
256
+
257
+ import { capabilitiesFromProjectMd } from './stack-capabilities.mjs';
258
+ import { runCodexExec, detectCodex } from './codex-exec.mjs';
259
+
260
+ export const SECOND_OPINION_PROVIDERS = Object.freeze(['codex', 'openrouter', 'none']);
261
+
262
+ /**
263
+ * FOUR states, and the fourth is the reason this function exists:
264
+ *
265
+ * declared the project chose a provider and it can run
266
+ * none the project decided: no second opinion
267
+ * undeclared nobody has said — NOT the same as none
268
+ * unavailable the project chose codex and there is no working codex here
269
+ *
270
+ * A caller that folds `unavailable` into `none` makes an absent reviewer look
271
+ * like a decision not to review.
272
+ *
273
+ * @param {{projectMd?:string, codex?:object, env?:object}} o
274
+ * `codex` is the shape of `codexStatusFrom`; pass it to stay pure.
275
+ */
276
+ export function resolveSecondOpinion({ projectMd = '', codex = null, env = process.env } = {}) {
277
+ const cap = capabilitiesFromProjectMd(projectMd).map.second_opinion ?? { state: 'undeclared', tool: null };
278
+ if (cap.state === 'none') {
279
+ return { state: 'none', provider: 'none', why: 'the project declared second_opinion: none' };
280
+ }
281
+ if (cap.state === 'undeclared') {
282
+ return { state: 'undeclared', provider: null, why: 'second_opinion is not declared in PROJECT.md capabilities — not declared is not none' };
283
+ }
284
+ const provider = String(cap.tool).toLowerCase();
285
+ if (!SECOND_OPINION_PROVIDERS.includes(provider)) {
286
+ return { state: 'unavailable', provider, why: `second_opinion: ${provider} is not a provider this plugin knows (${SECOND_OPINION_PROVIDERS.join(', ')})` };
287
+ }
288
+ if (provider === 'codex') {
289
+ const c = codex ?? detectCodex();
290
+ if (c.state !== 'available') return { state: 'unavailable', provider, why: c.why, codex: c };
291
+ return { state: 'declared', provider, why: '', codex: c };
292
+ }
293
+ if (provider === 'openrouter') {
294
+ if (!env.OPENROUTER_API_KEY) return { state: 'unavailable', provider, why: 'second_opinion: openrouter needs OPENROUTER_API_KEY' };
295
+ return { state: 'declared', provider, why: '' };
296
+ }
297
+ return { state: 'none', provider: 'none', why: '' };
298
+ }
299
+
300
+ /**
301
+ * The judge's `ask(question, allowed) → word` shape, backed by Codex.
302
+ *
303
+ * Same contract as `routerAsk` above, so `judge()` cannot tell them apart —
304
+ * which is the point: a second judge is only a second judge if it is
305
+ * interchangeable at the call site. Returns '' on anything that is not an
306
+ * answer, as routerAsk does; the caller already treats '' as "unparsed".
307
+ */
308
+ export function codexAsk(cwd, { model = null, timeoutMs = 120_000, bin = 'codex' } = {}) {
309
+ return async (question, allowed) => {
310
+ const prompt = `${question}\nAnswer with exactly one word from: ${allowed.join(', ')}. No explanation.`;
311
+ const r = await runCodexExec({ prompt, cwd, model, timeoutMs, bin });
312
+ return r.state === 'ok' ? r.text : '';
313
+ };
314
+ }
315
+
316
+ /**
317
+ * A full review turn through Codex: system + user prompt in, text + usage out.
318
+ * The three non-ok states pass through untouched so the reviewer can say
319
+ * "skipped: unreadable" rather than "PASS".
320
+ */
321
+ export async function codexReview({ system, user, cwd, model = null, timeoutMs = 300_000, bin = 'codex' }) {
322
+ const prompt = `${system}\n\n${user}`;
323
+ const r = await runCodexExec({ prompt, cwd, model, timeoutMs, bin });
324
+ return { ...r, model: r.model ?? model ?? null };
325
+ }
@@ -0,0 +1,206 @@
1
+ /**
2
+ * stack-capabilities — what THIS project's operational tools actually are.
3
+ *
4
+ * `l3-support` carries a routing table with thirteen alert sources: grafana,
5
+ * datadog, cloudwatch, eks, argocd, sentry, postgres, kafka, airflow, vercel,
6
+ * betterstack, mongo, and a generic row. It is a good table. What it cannot say
7
+ * is which of those thirteen rows is THIS project — so the agent infers the
8
+ * stack from whatever the alert happened to come from, at the moment somebody is
9
+ * being paged, and its generic fallback row guesses Grafana.
10
+ *
11
+ * `PROJECT.md` already carries a `stack:` line, but it is prose for humans
12
+ * ("TypeScript / Node.js 22 / Cloudflare Workers"). It names no log store, no
13
+ * pager, no error tracker. `stack-baseline` pins what a NEW product should be
14
+ * built with; `observability-baseline` wires it at scaffold time. Neither
15
+ * describes what an existing project has connected today.
16
+ *
17
+ * Borrowed from anthropics/oncall-kit, whose rule is that no skill names a
18
+ * vendor: skills refer to CAPABILITIES, and one file maps each capability to the
19
+ * tool the team actually has. Swap Datadog for Grafana and no skill changes.
20
+ *
21
+ * The vocabulary is deliberately small and closed. An open-ended map becomes a
22
+ * place to write anything, and then nothing can be resolved against it.
23
+ *
24
+ * Declared in PROJECT.md:
25
+ *
26
+ * capabilities:
27
+ * logs: grafana-loki
28
+ * metrics: grafana
29
+ * errors: sentry
30
+ * pager: none
31
+ *
32
+ * THREE STATES PER CAPABILITY, never two. `undeclared` and `none` are different
33
+ * answers: the first means nobody has said, the second means the project has
34
+ * deliberately decided it has no pager. An agent that treats "nobody said" as
35
+ * "there is none" stops looking for something that exists.
36
+ */
37
+
38
+ /** The closed vocabulary. Adding one is a deliberate edit, not a free-text key. */
39
+ export const CAPABILITIES = Object.freeze([
40
+ 'logs', // where log lines are searched
41
+ 'metrics', // time-series and dashboards
42
+ 'traces', // distributed traces
43
+ 'errors', // exception capture and grouping
44
+ 'alerts', // what fires, and where its rules live
45
+ 'pager', // who gets woken
46
+ 'deploys', // what shipped, and when
47
+ 'code-host', // where the diff and the PR live
48
+ // Not an incident tool: which harness gives the SECOND opinion in review and
49
+ // verification — `codex` (a different model family, through `codex exec`),
50
+ // `openrouter`, or `none`. It lives in this vocabulary rather than in a new
51
+ // file because the three-state rule is the whole point: a project that has
52
+ // not said is not a project that has turned it off.
53
+ 'second_opinion',
54
+ ]);
55
+
56
+ /** `none` is a decision. Anything else is a tool name we pass through verbatim. */
57
+ const NONE = 'none';
58
+
59
+ /**
60
+ * @returns {{state:'declared'|'none'|'undeclared', tool:string|null}}
61
+ */
62
+ function stateOf(raw) {
63
+ if (raw == null) return { state: 'undeclared', tool: null };
64
+ const v = String(raw).trim();
65
+ if (!v) return { state: 'undeclared', tool: null };
66
+ if (v.toLowerCase() === NONE) return { state: NONE, tool: null };
67
+ return { state: 'declared', tool: v };
68
+ }
69
+
70
+ /**
71
+ * Read the capability block out of a PROJECT.md body.
72
+ *
73
+ * Parsed with the same shape as `levelFromProjectMd` — a top-level key followed
74
+ * by indented `name: value` lines — rather than by pulling in a YAML parser the
75
+ * board's zero-dependency rule forbids.
76
+ *
77
+ * @returns {{map: Record<string,{state:string,tool:string|null}>, declaredCount:number, unknownKeys:string[]}}
78
+ */
79
+ export function capabilitiesFromProjectMd(text = '') {
80
+ const body = String(text);
81
+ const start = body.match(/^capabilities:[ \t]*$/m);
82
+ const raw = {};
83
+ const unknownKeys = [];
84
+
85
+ if (start) {
86
+ const after = body.slice(start.index + start[0].length);
87
+ for (const line of after.split('\n')) {
88
+ if (/^\S/.test(line)) break; // dedent ends the block
89
+ const m = line.match(/^[ \t]+([a-z][a-z0-9_-]*)\s*:\s*(.*)$/i);
90
+ if (!m) continue;
91
+ const key = m[1].toLowerCase();
92
+ // An unrecognised key is REPORTED, not dropped. A capability nobody reads
93
+ // because it was misspelled is the same as one nobody wrote, except that
94
+ // the author believes it is there.
95
+ if (!CAPABILITIES.includes(key)) { unknownKeys.push(key); continue; }
96
+ raw[key] = m[2];
97
+ }
98
+ }
99
+
100
+ const map = {};
101
+ let declaredCount = 0;
102
+ for (const cap of CAPABILITIES) {
103
+ map[cap] = stateOf(raw[cap]);
104
+ if (map[cap].state === 'declared') declaredCount++;
105
+ }
106
+ return { map, declaredCount, unknownKeys };
107
+ }
108
+
109
+ /**
110
+ * A line an agent can act on, for one capability.
111
+ *
112
+ * The `undeclared` wording matters: it must send the reader to find out, not let
113
+ * them conclude there is nothing there. This is the sentence l3-support reads at
114
+ * 3am, so it says what to do rather than what is missing.
115
+ */
116
+ export function describeCapability(cap, entry) {
117
+ if (entry.state === 'declared') return `${cap}: ${entry.tool}`;
118
+ if (entry.state === NONE) return `${cap}: none — this project has decided it has no ${cap}`;
119
+ return `${cap}: not declared — do not assume there is none; ask, or fall back to the alert-source routing table`;
120
+ }
121
+
122
+ /** The whole block, for injection into an agent brief. */
123
+ export function describeCapabilities({ map, declaredCount, unknownKeys }) {
124
+ const lines = CAPABILITIES.map((c) => ` ${describeCapability(c, map[c])}`);
125
+ const head = declaredCount === 0
126
+ ? 'This project declares NO operational capabilities. Nothing below is known;'
127
+ + ' route by alert source and say so rather than implying the stack was checked.'
128
+ : `This project declares ${declaredCount} of ${CAPABILITIES.length} capabilities.`;
129
+ const warn = unknownKeys.length
130
+ ? [` ⚠ unrecognised capability key(s) in PROJECT.md, ignored: ${unknownKeys.join(', ')}`]
131
+ : [];
132
+ return [head, ...lines, ...warn].join('\n');
133
+ }
134
+
135
+ // ── CLI ──────────────────────────────────────────────────────────────────────
136
+ // Read by l3-support at the top of an incident, the same way /recall reaches
137
+ // memory-search: a block in the agent's own prompt, no new plumbing.
138
+ //
139
+ // node scripts/lib/stack-capabilities.mjs [--cwd DIR] [--json]
140
+ //
141
+ // Exit 0 always. A project with nothing declared is a fact to report, not an
142
+ // error to fail on — failing here would make an unconfigured project unable to
143
+ // run an incident, which is exactly when you need the agent most.
144
+ if (import.meta.url === `file://${process.argv[1]}`) {
145
+ const { readFileSync } = await import('node:fs');
146
+ const { join } = await import('node:path');
147
+ const args = process.argv.slice(2);
148
+ const cwd = args.includes('--cwd') ? args[args.indexOf('--cwd') + 1] : process.cwd();
149
+ const asJson = args.includes('--json');
150
+
151
+ let text = '', read = 'ok';
152
+ try { text = readFileSync(join(cwd, '.great_cto', 'PROJECT.md'), 'utf8'); }
153
+ catch (err) { read = err.code ?? String(err.message); }
154
+
155
+ const result = capabilitiesFromProjectMd(text);
156
+ if (asJson) {
157
+ console.log(JSON.stringify({ read, ...result }, null, 2));
158
+ } else if (read !== 'ok') {
159
+ // Distinct from "declares nothing": there was no PROJECT.md to read at all.
160
+ console.log(`no PROJECT.md under ${cwd}/.great_cto (${read}) — the project's capabilities are unknown, not absent.`);
161
+ } else {
162
+ console.log(describeCapabilities(result));
163
+ }
164
+ }
165
+
166
+ /**
167
+ * Set one capability in a PROJECT.md body, returning the new text and what was
168
+ * there before. Pure, so the board's POST can be tested without a filesystem.
169
+ *
170
+ * The previous value is returned, not just the new one: a setting that
171
+ * silently replaced another is a change the operator cannot see they made —
172
+ * the same rule /api/agent-budgets follows.
173
+ *
174
+ * Creates the `capabilities:` block at the end when it is missing. `null` as
175
+ * the value removes the key (back to `undeclared`), which is different from
176
+ * writing `none`.
177
+ *
178
+ * @returns {{text:string, previous:string|null, created:boolean}}
179
+ */
180
+ export function upsertCapability(text, key, value) {
181
+ if (!CAPABILITIES.includes(key)) throw new Error(`not a capability: ${key}`);
182
+ if (value != null && !/^[A-Za-z0-9_.:@/-]+$/.test(String(value))) throw new Error(`invalid value for ${key}`);
183
+ const body = String(text ?? '');
184
+ const lines = body.split('\n');
185
+ const start = lines.findIndex((l) => /^capabilities:[ \t]*$/.test(l));
186
+ const previous = capabilitiesFromProjectMd(body).map[key]?.tool
187
+ ?? (capabilitiesFromProjectMd(body).map[key]?.state === NONE ? NONE : null);
188
+
189
+ if (start === -1) {
190
+ if (value == null) return { text: body, previous, created: false };
191
+ const nl = body.endsWith('\n') || body === '' ? '' : '\n';
192
+ return { text: `${body}${nl}\ncapabilities:\n ${key}: ${value}\n`, previous, created: true };
193
+ }
194
+
195
+ let end = start + 1;
196
+ while (end < lines.length && /^[ \t]+\S/.test(lines[end])) end += 1;
197
+ const idx = lines.findIndex((l, i) => i > start && i < end && new RegExp(`^[ \\t]+${key}[ \\t]*:`).test(l));
198
+
199
+ if (value == null) {
200
+ if (idx !== -1) lines.splice(idx, 1);
201
+ return { text: lines.join('\n'), previous, created: false };
202
+ }
203
+ const line = ` ${key}: ${value}`;
204
+ if (idx !== -1) lines[idx] = line; else lines.splice(end, 0, line);
205
+ return { text: lines.join('\n'), previous, created: false };
206
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "great-cto",
3
- "version": "3.25.0",
3
+ "version": "3.26.0",
4
4
  "description": "One command install for the great_cto Claude Code plugin. Auto-detects your stack, picks the right archetype, bootstraps PROJECT.md.",
5
5
  "keywords": [
6
6
  "claude-code",